diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 853a189cc9..fbfcaa5c41 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,7 +2,7 @@ * @Percona-Lab/build-engineers /cloud/gcp-functions/ @egegunes @inelpandzic -/cloud/ @hors @tplavcic @nmarukovich @ptankov @jvpasinatto @eleo007 +/cloud/ @hors @tplavcic @nmarukovich @jvpasinatto @eleo007 /pmm/ @talhabinrizwan @ademidoff @puneet0191 /IaC/pmm.cd/ @talhabinrizwan @ademidoff /pxb/ @mchawla16 @eleo007 @mohitj1988 @kaushikpuneet07 @panchal-yash @Percona-Lab/build-engineers @@ -14,3 +14,4 @@ /pdmdb/ @olexandr-havryliak @sandraromanchenko @Percona-Lab/build-engineers /pdps/ @eleo007 @kaushikpuneet07 @Percona-Lab/build-engineers /pdpxc/ @eleo007 @kaushikpuneet07 @Percona-Lab/build-engineers +/pml/ @olexandr-havryliak @sandraromanchenko @keithquinnpercona @Percona-Lab/build-engineers diff --git a/IaC/pmm.cd/init.groovy.d/htz.cloud.groovy b/IaC/pmm.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..5a9c8b60f5 --- /dev/null +++ b/IaC/pmm.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,188 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "pmm-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/ps3.cd/init.groovy.d/htz.cloud.groovy b/IaC/ps3.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..dc232edb35 --- /dev/null +++ b/IaC/ps3.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,188 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "ps3-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/ps57.cd/init.groovy.d/htz.cloud.groovy b/IaC/ps57.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..1712b5d300 --- /dev/null +++ b/IaC/ps57.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,188 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "ps57-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/ps80.cd/init.groovy.d/htz.cloud.groovy b/IaC/ps80.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..9f46e8fcde --- /dev/null +++ b/IaC/ps80.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,188 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "ps80-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/ps80.cd/init.groovy.d/matrix.groovy b/IaC/ps80.cd/init.groovy.d/matrix.groovy index 33309cfece..a82e3424ce 100644 --- a/IaC/ps80.cd/init.groovy.d/matrix.groovy +++ b/IaC/ps80.cd/init.groovy.d/matrix.groovy @@ -149,6 +149,7 @@ authz_strategy_config = [ 'percona*QA-ps': ['Overall Read','Agent Configure','Agent Delete','Agent Create','Agent Disconnect','Agent Connect','Agent Build','Agent Provision','Run Delete','Run Update','Run Replay','Job Create','Job Delete','Job Configure','Job Read','Job Discover','Job Build','Job Workspace','Job Cancel','Job Move','View Create','View Delete','View Configure','View Read'], 'percona*qa-distro': ['Overall Read','Agent Configure','Agent Delete','Agent Create','Agent Disconnect','Agent Connect','Agent Build','Agent Provision','Run Delete','Run Update','Run Replay','Job Create','Job Delete','Job Configure','Job Read','Job Discover','Job Build','Job Workspace','Job Cancel','Job Move','View Create','View Delete','View Configure','View Read'], 'percona*doc': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], + 'JNKPercona': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], 'percona*external-contractors-ps': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], ] ] diff --git a/IaC/psmdb.cd/init.groovy.d/htz.cloud.groovy b/IaC/psmdb.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..243fd80920 --- /dev/null +++ b/IaC/psmdb.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,188 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "psmdb-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/psmdb.cd/init.groovy.d/matrix.groovy b/IaC/psmdb.cd/init.groovy.d/matrix.groovy index 0723f5e6d4..8ad93fc13d 100644 --- a/IaC/psmdb.cd/init.groovy.d/matrix.groovy +++ b/IaC/psmdb.cd/init.groovy.d/matrix.groovy @@ -148,6 +148,7 @@ authz_strategy_config = [ 'percona*dev-psmdb': ['Overall Read','Agent Configure','Agent Delete','Agent Create','Agent Disconnect','Agent Connect','Agent Build','Agent Provision','Run Delete','Run Update','Run Replay','Job Create','Job Delete','Job Configure','Job Read','Job Discover','Job Build','Job Workspace','Job Cancel','Job Move','View Create','View Delete','View Configure','View Read'], 'percona*qa-distro': ['Overall Read','Agent Configure','Agent Delete','Agent Create','Agent Disconnect','Agent Connect','Agent Build','Agent Provision','Run Delete','Run Update','Run Replay','Job Create','Job Delete','Job Configure','Job Read','Job Discover','Job Build','Job Workspace','Job Cancel','Job Move','View Create','View Delete','View Configure','View Read'], 'percona*doc': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], + 'JNKPercona': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], ] ] diff --git a/IaC/pxb.cd/init.groovy.d/htz.cloud.groovy b/IaC/pxb.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..17f9b285ac --- /dev/null +++ b/IaC/pxb.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,188 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "pxb-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/pxc.cd/init.groovy.d/htz.cloud.groovy b/IaC/pxc.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..af7919c977 --- /dev/null +++ b/IaC/pxc.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,188 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "pxc-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/pxc.cd/init.groovy.d/matrix.groovy b/IaC/pxc.cd/init.groovy.d/matrix.groovy index 33309cfece..a82e3424ce 100644 --- a/IaC/pxc.cd/init.groovy.d/matrix.groovy +++ b/IaC/pxc.cd/init.groovy.d/matrix.groovy @@ -149,6 +149,7 @@ authz_strategy_config = [ 'percona*QA-ps': ['Overall Read','Agent Configure','Agent Delete','Agent Create','Agent Disconnect','Agent Connect','Agent Build','Agent Provision','Run Delete','Run Update','Run Replay','Job Create','Job Delete','Job Configure','Job Read','Job Discover','Job Build','Job Workspace','Job Cancel','Job Move','View Create','View Delete','View Configure','View Read'], 'percona*qa-distro': ['Overall Read','Agent Configure','Agent Delete','Agent Create','Agent Disconnect','Agent Connect','Agent Build','Agent Provision','Run Delete','Run Update','Run Replay','Job Create','Job Delete','Job Configure','Job Read','Job Discover','Job Build','Job Workspace','Job Cancel','Job Move','View Create','View Delete','View Configure','View Read'], 'percona*doc': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], + 'JNKPercona': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], 'percona*external-contractors-ps': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], ] ] diff --git a/IaC/rel.cd/init.groovy.d/htz.cloud.groovy b/IaC/rel.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..74eef04fc6 --- /dev/null +++ b/IaC/rel.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,188 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "rel-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/cloud/jenkins/pgo_operator_eks_latest.groovy b/cloud/jenkins/pgo_operator_eks_latest.groovy index c25e55a1f9..ffee85708a 100644 --- a/cloud/jenkins/pgo_operator_eks_latest.groovy +++ b/cloud/jenkins/pgo_operator_eks_latest.groovy @@ -31,7 +31,9 @@ void prepareNode() { } if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/pgo_operator_eks_version.groovy b/cloud/jenkins/pgo_operator_eks_version.groovy index 6a3105ebce..76c419901e 100644 --- a/cloud/jenkins/pgo_operator_eks_version.groovy +++ b/cloud/jenkins/pgo_operator_eks_version.groovy @@ -31,7 +31,9 @@ void prepareNode() { } if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/pgo_operator_gke_latest.groovy b/cloud/jenkins/pgo_operator_gke_latest.groovy index 4b3b7845ad..455be73104 100644 --- a/cloud/jenkins/pgo_operator_gke_latest.groovy +++ b/cloud/jenkins/pgo_operator_gke_latest.groovy @@ -48,7 +48,7 @@ EOF } if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d '.' -f 1,2", , returnStdout: true).trim() + USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d '.' -f 1,2", , returnStdout: true).trim() } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/pgo_operator_gke_version.groovy b/cloud/jenkins/pgo_operator_gke_version.groovy index d23429bdf4..89b5758979 100644 --- a/cloud/jenkins/pgo_operator_gke_version.groovy +++ b/cloud/jenkins/pgo_operator_gke_version.groovy @@ -48,7 +48,7 @@ EOF } if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d '.' -f 1,2", , returnStdout: true).trim() + USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d '.' -f 1,2", , returnStdout: true).trim() } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/ps_operator_eks_latest.groovy b/cloud/jenkins/ps_operator_eks_latest.groovy index 983edc7008..617d44a856 100644 --- a/cloud/jenkins/ps_operator_eks_latest.groovy +++ b/cloud/jenkins/ps_operator_eks_latest.groovy @@ -29,7 +29,9 @@ void prepareNode() { void prepareSources() { if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/ps_operator_eks_version.groovy b/cloud/jenkins/ps_operator_eks_version.groovy index 057c80e912..86cd5de840 100644 --- a/cloud/jenkins/ps_operator_eks_version.groovy +++ b/cloud/jenkins/ps_operator_eks_version.groovy @@ -29,7 +29,9 @@ void prepareNode() { void prepareSources() { if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/ps_operator_gke_latest.groovy b/cloud/jenkins/ps_operator_gke_latest.groovy index 72aa948a14..7722188f3b 100644 --- a/cloud/jenkins/ps_operator_gke_latest.groovy +++ b/cloud/jenkins/ps_operator_gke_latest.groovy @@ -46,7 +46,7 @@ EOF void prepareSources() { if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d- -f1", , returnStdout: true).trim() + USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d- -f1", , returnStdout: true).trim() } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/ps_operator_gke_version.groovy b/cloud/jenkins/ps_operator_gke_version.groovy index c81701fa04..2ac3eeacde 100644 --- a/cloud/jenkins/ps_operator_gke_version.groovy +++ b/cloud/jenkins/ps_operator_gke_version.groovy @@ -46,7 +46,7 @@ EOF void prepareSources() { if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d- -f1", , returnStdout: true).trim() + USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d- -f1", , returnStdout: true).trim() } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/psmdb_operator_eks_latest.groovy b/cloud/jenkins/psmdb_operator_eks_latest.groovy index 0c8052706c..f546efec2b 100644 --- a/cloud/jenkins/psmdb_operator_eks_latest.groovy +++ b/cloud/jenkins/psmdb_operator_eks_latest.groovy @@ -70,7 +70,9 @@ void prepareNode() { """ if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/psmdb_operator_eks_version.groovy b/cloud/jenkins/psmdb_operator_eks_version.groovy index 850598dcdd..b36191c1ab 100644 --- a/cloud/jenkins/psmdb_operator_eks_version.groovy +++ b/cloud/jenkins/psmdb_operator_eks_version.groovy @@ -70,7 +70,9 @@ void prepareNode() { """ if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/psmdb_operator_gke_latest.groovy b/cloud/jenkins/psmdb_operator_gke_latest.groovy index 199b2e0645..e83a07d8bf 100644 --- a/cloud/jenkins/psmdb_operator_gke_latest.groovy +++ b/cloud/jenkins/psmdb_operator_gke_latest.groovy @@ -90,7 +90,7 @@ EOF } if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d- -f1", , returnStdout: true).trim() + USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d- -f1", , returnStdout: true).trim() } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/psmdb_operator_gke_version.groovy b/cloud/jenkins/psmdb_operator_gke_version.groovy index 01fbd469c2..5d03f3070d 100644 --- a/cloud/jenkins/psmdb_operator_gke_version.groovy +++ b/cloud/jenkins/psmdb_operator_gke_version.groovy @@ -90,7 +90,7 @@ EOF } if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d- -f1", , returnStdout: true).trim() + USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d- -f1", , returnStdout: true).trim() } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/pxco-eks.groovy b/cloud/jenkins/pxco-eks.groovy index 12e60d534a..b0de107790 100644 --- a/cloud/jenkins/pxco-eks.groovy +++ b/cloud/jenkins/pxco-eks.groovy @@ -63,7 +63,9 @@ void prepareNode() { """ if ("$PLATFORM_VER" == "latest") { - PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } if ("$IMAGE_PXC") { diff --git a/cloud/jenkins/pxco-gke.groovy b/cloud/jenkins/pxco-gke.groovy index 4c5a84d2ae..09422860dc 100644 --- a/cloud/jenkins/pxco-gke.groovy +++ b/cloud/jenkins/pxco-gke.groovy @@ -83,7 +83,7 @@ EOF } if ("$PLATFORM_VER" == "latest") { - PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d- -f1", returnStdout: true).trim() + PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d- -f1", returnStdout: true).trim() } if ("$IMAGE_PXC") { diff --git a/orchestrator/jenkins/orchestrator-packaging.groovy b/orchestrator/jenkins/orchestrator-packaging.groovy index 8b32efd635..8ba2c7e9a7 100644 --- a/orchestrator/jenkins/orchestrator-packaging.groovy +++ b/orchestrator/jenkins/orchestrator-packaging.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -30,9 +30,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/orchestrator.git', description: 'URL for orchestrator repository', @@ -89,36 +93,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build orchestrator generic source packages') { parallel { stage('Build orchestrator generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build orchestrator generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -127,184 +131,184 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } diff --git a/orchestrator/jenkins/orchestrator-packaging.yml b/orchestrator/jenkins/orchestrator-packaging.yml index 91a009371d..50aef7d69a 100644 --- a/orchestrator/jenkins/orchestrator-packaging.yml +++ b/orchestrator/jenkins/orchestrator-packaging.yml @@ -1,5 +1,5 @@ - job: - name: orchestrator-RELEASE + name: hetzner-orchestrator-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: orchestrator/jenkins/orchestrator-packaging.groovy diff --git a/pbm/hetzner-pbm-functional-tests-full.groovy b/pbm/hetzner-pbm-functional-tests-full.groovy new file mode 100644 index 0000000000..0fbe2efb95 --- /dev/null +++ b/pbm/hetzner-pbm-functional-tests-full.groovy @@ -0,0 +1,99 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label 'launcher-x64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + string(name: 'PBM_BRANCH', defaultValue: 'main', description: 'PBM branch or commit') + string(name: 'GO_VER', defaultValue: 'bullseye', description: 'GOLANG docker image for building PBM from sources') + choice(name: 'instance', choices: ['docker-x64','docker-aarch64'], description: 'Instance type for running tests') + string(name: 'TESTING_BRANCH', defaultValue: 'main', description: 'psmdb-testing repo branch') + string(name: 'PYTEST_PARAMS', defaultValue: '', description: 'Extra args passed to pytest') + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PBM_BRANCH}" + } + } + } + stage ('Run tests') { + matrix { + agent { + label "${params.instance}" + } + axes { + axis { + name 'TEST' + values 'logical', 'physical', 'incremental', 'external', 'load' + } + axis { + name 'PSMDB' + values '6.0', '7.0', '8.0' + } + } + stages { + stage ('Run tests') { + steps { + withCredentials([string(credentialsId: 'olexandr_zephyr_token', variable: 'ZEPHYR_TOKEN')]) { + sh """ + docker kill \$(docker ps -a -q) || true + docker rm \$(docker ps -a -q) || true + docker rmi -f \$(docker images -q | uniq) || true + sudo rm -rf ./* + if [ ! -f "/usr/local/bin/docker-compose" ] ; then + if [ ${params.instance} = "docker-aarch64" ]; then + sudo curl -SL https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-linux-aarch64 -o /usr/local/bin/docker-compose + else + sudo curl -SL https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose + fi + sudo chmod +x /usr/local/bin/docker-compose + fi + """ + git poll: false, branch: params.TESTING_BRANCH, url: 'https://github.com/Percona-QA/psmdb-testing.git' + sh """ + cd pbm-functional/pytest + PSMDB=perconalab/percona-server-mongodb:${PSMDB} docker-compose build --no-cache + docker-compose up -d + docker-compose run test pytest -s --junitxml=junit.xml -k ${TEST} ${params.PYTEST_PARAMS} || true + docker-compose down -v --remove-orphans + curl -H "Content-Type:multipart/form-data" -H "Authorization: Bearer ${ZEPHYR_TOKEN}" -F "file=@junit.xml;type=application/xml" 'https://api.zephyrscale.smartbear.com/v2/automations/executions/junit?projectKey=PBM' -F 'testCycle={"name":"${JOB_NAME}-${BUILD_NUMBER}","customFields": { "PBM branch": "${PBM_BRANCH}","PSMDB docker image": "percona/percona-server-mongodb:${PSMDB}","instance": "${instance}"}};type=application/json' -i || true + """ + } + } + post { + always { + junit testResults: "**/junit.xml", keepLongStdio: true, allowEmptyResults: true, skipPublishingChecks: true + sh """ + docker kill \$(docker ps -a -q) || true + docker rm \$(docker ps -a -q) || true + docker rmi -f \$(docker images -q | uniq) || true + sudo rm -rf ./* + """ + } + } + } + } + } + } + } + post { + success { + slackNotify("#mongodb_autofeed", "#00FF00", "[${JOB_NAME}]: PBM ${PBM_BRANCH} - all tests passed [${BUILD_URL}testReport/]") + } + unstable { + slackNotify("#mongodb_autofeed", "#F6F930", "[${JOB_NAME}]: PBM ${PBM_BRANCH} - some tests failed [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#mongodb_autofeed", "#FF0000", "[${JOB_NAME}]: PBM ${PBM_BRANCH} - unexpected failure [${BUILD_URL}]") + } + } +} diff --git a/pbm/hetzner-pbm-functional-tests-full.yml b/pbm/hetzner-pbm-functional-tests-full.yml new file mode 100644 index 0000000000..d21a601da6 --- /dev/null +++ b/pbm/hetzner-pbm-functional-tests-full.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pbm-functional-tests-full + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pbm/hetzner-pbm-functional-tests-full.groovy diff --git a/pbm/hetzner-pbm-functional-tests.groovy b/pbm/hetzner-pbm-functional-tests.groovy new file mode 100644 index 0000000000..1bf634cc11 --- /dev/null +++ b/pbm/hetzner-pbm-functional-tests.groovy @@ -0,0 +1,99 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label 'launcher-x64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + string(name: 'PBM_BRANCH', defaultValue: 'main', description: 'PBM branch') + string(name: 'PSMDB', defaultValue: 'percona/percona-server-mongodb', description: 'PSMDB docker image') + string(name: 'GO_VER', defaultValue: 'bullseye', description: 'GOLANG docker image for building PBM from sources') + choice(name: 'instance', choices: ['docker-x64','docker-aarch64'], description: 'Instance type for running tests') + string(name: 'TESTING_BRANCH', defaultValue: 'main', description: 'psmdb-testing repo branch') + string(name: 'PYTEST_PARAMS', defaultValue: '', description: 'Extra args passed to pytest') + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PBM_BRANCH}-${params.PSMDB}" + } + } + } + stage ('Run tests') { + matrix { + agent { + label "${params.instance}" + } + axes { + axis { + name 'TEST' + values 'logical', 'physical', 'incremental', 'external', 'load' + } + } + stages { + stage ('Run tests') { + steps { + withCredentials([string(credentialsId: 'olexandr_zephyr_token', variable: 'ZEPHYR_TOKEN')]) { + sh """ + docker kill \$(docker ps -a -q) || true + docker rm \$(docker ps -a -q) || true + docker rmi -f \$(docker images -q | uniq) || true + sudo rm -rf ./* + + if [ ! -f "/usr/local/bin/docker-compose" ] ; then + if [ ${params.instance} = "docker-aarch64" ]; then + sudo curl -SL https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-linux-aarch64 -o /usr/local/bin/docker-compose + else + sudo curl -SL https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose + fi + sudo chmod +x /usr/local/bin/docker-compose + fi + + git clone https://github.com/Percona-QA/psmdb-testing + cd psmdb-testing + git checkout ${params.TESTING_BRANCH} + + cd pbm-functional/pytest + docker-compose build + docker-compose up -d + docker-compose run test pytest -s --junitxml=junit.xml -k ${TEST} ${params.PYTEST_PARAMS} || true + docker-compose down -v --remove-orphans + curl -H "Content-Type:multipart/form-data" -H "Authorization: Bearer ${ZEPHYR_TOKEN}" -F "file=@junit.xml;type=application/xml" 'https://api.zephyrscale.smartbear.com/v2/automations/executions/junit?projectKey=PBM' -F 'testCycle={"name":"${JOB_NAME}-${BUILD_NUMBER}","customFields": { "PBM branch": "${PBM_BRANCH}","PSMDB docker image": "${PSMDB}","instance": "${instance}"}};type=application/json' -i || true + """ + } + } + post { + always { + junit testResults: "**/junit.xml", keepLongStdio: true, allowEmptyResults: true, skipPublishingChecks: true + sh """ + docker kill \$(docker ps -a -q) || true + docker rm \$(docker ps -a -q) || true + docker rmi -f \$(docker images -q | uniq) || true + sudo rm -rf ./* + """ + } + } + } + } + } + } + } + post { + success { + slackNotify("#mongodb_autofeed", "#00FF00", "[${JOB_NAME}]: PBM ${PBM_BRANCH} with ${PSMDB} - all tests passed") + } + unstable { + slackNotify("#mongodb_autofeed", "#F6F930", "[${JOB_NAME}]: PBM ${PBM_BRANCH} with ${PSMDB} - some tests failed [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#mongodb_autofeed", "#FF0000", "[${JOB_NAME}]: PBM ${PBM_BRANCH} with ${PSMDB} - unexpected failure [${BUILD_URL}]") + } + } +} diff --git a/pbm/hetzner-pbm-functional-tests.yml b/pbm/hetzner-pbm-functional-tests.yml new file mode 100644 index 0000000000..4749f20161 --- /dev/null +++ b/pbm/hetzner-pbm-functional-tests.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pbm-functional-tests + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pbm/hetzner-pbm-functional-tests.groovy diff --git a/pbm/hetzner-pbm-site-check.groovy b/pbm/hetzner-pbm-site-check.groovy new file mode 100644 index 0000000000..8a6267f84b --- /dev/null +++ b/pbm/hetzner-pbm-site-check.groovy @@ -0,0 +1,68 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label 'docker-x64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + string( + defaultValue: '2.3.0', + description: 'PBM Version for tests', + name: 'PBM_VERSION') + string( + defaultValue: 'main', + description: 'Branch for testing repository', + name: 'TESTING_BRANCH') + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PBM_VERSION}" + } + } + } + stage('Checkout') { + steps { + deleteDir() + git poll: false, branch: TESTING_BRANCH, url: 'https://github.com/Percona-QA/psmdb-testing.git' + } + } + stage('Test') { + steps { + script { + sh """ + cd site_checks + docker run --env PBM_VERSION=${params.PBM_VERSION} --rm -v `pwd`:/tmp -w /tmp python bash -c 'pip3 install requests pytest setuptools && pytest -s --junitxml=junit.xml test_pbm.py || [ \$? = 1 ] ' + """ + } + } + } + } + post { + success { + slackNotify("#mongodb_autofeed", "#00FF00", "[${JOB_NAME}]: checking packages on the main site for PBM ${PBM_VERSION} - ok [${BUILD_URL}testReport/]") + } + unstable { + slackNotify("#mongodb_autofeed", "#F6F930", "[${JOB_NAME}]: checking packages on the main site for PBM ${PBM_VERSION} - some links are broken [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#mongodb_autofeed", "#FF0000", "[${JOB_NAME}]: checking packages on the main site for PBM ${PBM_VERSION} - failed [${BUILD_URL}]" ) + } + always { + script { + junit testResults: "**/junit.xml", keepLongStdio: true, allowEmptyResults: true, skipPublishingChecks: true + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } + } +} diff --git a/pbm/hetzner-pbm-site-check.yml b/pbm/hetzner-pbm-site-check.yml new file mode 100644 index 0000000000..07b2a3507f --- /dev/null +++ b/pbm/hetzner-pbm-site-check.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pbm-site-check + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pbm/hetzner-pbm-site-check.groovy diff --git a/pbm/jenkins/get-pbm-branches.groovy b/pbm/jenkins/get-pbm-branches.groovy index f9b256f199..693d80c057 100644 --- a/pbm/jenkins/get-pbm-branches.groovy +++ b/pbm/jenkins/get-pbm-branches.groovy @@ -1,13 +1,17 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-backup-mongodb.git', description: 'URL for percona-mongodb-backup repository', @@ -22,50 +26,50 @@ pipeline { stages { stage('Get release branches') { steps { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sh """ - EC=0 - aws s3 ls s3://percona-jenkins-artifactory/percona-backup-mongodb/branch_commit_id.properties || EC=\$? + script { + String S3_STASH = (params.CLOUD == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (params.CLOUD == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + EC=0 + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 ls s3://percona-jenkins-artifactory/percona-backup-mongodb/branch_commit_id.properties ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 || EC=\$? - if [ \${EC} = 1 ]; then - LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release\\* | tail -1) - BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) - COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) + if [ \${EC} = 1 ]; then + LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release\\* | tail -1) + BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) + COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) - echo "BRANCH_NAME=\${BRANCH_NAME}" > branch_commit_id.properties - echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id.properties + echo "BRANCH_NAME=\${BRANCH_NAME}" > branch_commit_id.properties + echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id.properties - aws s3 cp branch_commit_id.properties s3://percona-jenkins-artifactory/percona-backup-mongodb/ - echo "START_NEW_BUILD=NO" > startBuild - else - aws s3 cp s3://percona-jenkins-artifactory/percona-backup-mongodb/branch_commit_id.properties . - source branch_commit_id.properties + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp branch_commit_id.properties s3://percona-jenkins-artifactory/percona-backup-mongodb/ ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 + echo "START_NEW_BUILD=NO" > startBuild + else + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp s3://percona-jenkins-artifactory/percona-backup-mongodb/branch_commit_id.properties . ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 + source branch_commit_id.properties - LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release\\* | tail -1) - LATEST_BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) - LATEST_COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) + LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release\\* | tail -1) + LATEST_BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) + LATEST_COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) - if [ "x\${COMMIT_ID}" != "x\${LATEST_COMMIT_ID}" ] || [ "x\${BRANCH_NAME}" != "x\${LATEST_BRANCH_NAME}" ]; then - echo "START_NEW_BUILD=YES" > startBuild - else - echo "START_NEW_BUILD=NO" > startBuild - fi + if [ "x\${COMMIT_ID}" != "x\${LATEST_COMMIT_ID}" ] || [ "x\${BRANCH_NAME}" != "x\${LATEST_BRANCH_NAME}" ]; then + echo "START_NEW_BUILD=YES" > startBuild + else + echo "START_NEW_BUILD=NO" > startBuild + fi - echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id.properties - echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id.properties - aws s3 cp branch_commit_id.properties s3://percona-jenkins-artifactory/percona-backup-mongodb/ - fi - """ - } - script { + echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id.properties + echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id.properties + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp branch_commit_id.properties s3://percona-jenkins-artifactory/percona-backup-mongodb/ ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 + fi + """ + } START_NEW_BUILD = sh(returnStdout: true, script: "source startBuild; echo \${START_NEW_BUILD}").trim() BRANCH_NAME = sh(returnStdout: true, script: "source branch_commit_id.properties; echo \${BRANCH_NAME}").trim() COMMIT_ID = sh(returnStdout: true, script: "source branch_commit_id.properties; echo \${COMMIT_ID}").trim() VERSION = sh(returnStdout: true, script: "source branch_commit_id.properties; echo \${BRANCH_NAME} | sed s:release\\-::").trim() } - } - } stage('Build needed') { when { @@ -78,8 +82,8 @@ pipeline { """ } slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") - build job: 'pbm-autobuild-RELEASE', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'VERSION', value: VERSION), string(name: 'COMPONENT', value: 'testing')] - build job: 'pbm-release-test-run', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: BRANCH_NAME), string(name: 'PBM_VERSION', value: VERSION)] + build job: 'hetzner-pbm-autobuild-RELEASE', parameters: [string(name: 'CLOUD', value: CLOUD), string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'VERSION', value: VERSION), string(name: 'COMPONENT', value: 'testing')] + build job: 'hetzner-pbm-release-test-run', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: BRANCH_NAME), string(name: 'PBM_VERSION', value: VERSION)] } } stage('Build skipped') { diff --git a/pbm/jenkins/get-pbm-branches.yml b/pbm/jenkins/get-pbm-branches.yml index e6b0065d57..23a503b6b9 100644 --- a/pbm/jenkins/get-pbm-branches.yml +++ b/pbm/jenkins/get-pbm-branches.yml @@ -1,5 +1,5 @@ - job: - name: pbm-autobuild + name: hetzner-pbm-autobuild project-type: pipeline description: | Do not edit this job through the web! @@ -9,7 +9,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pbm/jenkins/get-pbm-branches.groovy diff --git a/pbm/jenkins/percona-mongodb-backup.groovy b/pbm/jenkins/percona-mongodb-backup.groovy index 9ad4ef2c3a..77c2616962 100644 --- a/pbm/jenkins/percona-mongodb-backup.groovy +++ b/pbm/jenkins/percona-mongodb-backup.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,15 +29,19 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-backup-mongodb.git', description: 'URL for percona-mongodb-backup repository', name: 'GIT_REPO') string( - defaultValue: 'master', + defaultValue: 'dev', description: 'Tag/Branch for percona-mongodb-backup repository', name: 'GIT_BRANCH') string( @@ -49,7 +53,7 @@ pipeline { description: 'DEB release value', name: 'DEB_RELEASE') string( - defaultValue: '1.3.2', + defaultValue: '2.8.0', description: 'VERSION value', name: 'VERSION') string( @@ -57,9 +61,25 @@ pipeline { description: 'PBM repo name', name: 'PBM_REPO') choice( - choices: 'laboratory\ntesting\nexperimental', + choices: 'experimental\nlaboratory\ntesting\nexperimental', description: 'Repo component to push packages to', name: 'COMPONENT') + choice( + name: 'BUILD_DOCKER', + choices: ['true', 'false'], + description: 'Build and push Docker images (default: true)') + choice( + name: 'BUILD_PACKAGES', + choices: ['true', 'false'], + description: 'Build packages (default: true)') + choice( + name: 'TARGET_REPO', + choices: ['PerconaLab','DockerHub'], + description: 'Target repo for docker image, use DockerHub for release only') + choice( + name: 'PBM_REPO_TYPE', + choices: ['testing','release','experimental'], + description: 'Packages repo for docker images') } options { skipDefaultCheckout() @@ -69,8 +89,11 @@ pipeline { } stages { stage('Create PBM source tarball') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -88,179 +111,418 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PBM generic source packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { stage('Build PBM generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PBM generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PBM RPMs/DEBs/Binary tarballs') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8(aarch64)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9(x86_64)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Amazon Linux 2023') { + stage('Oracle Linux 9(aarch64)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Amazon Linux 2023(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("amazonlinux:2023", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Amazon Linux 2023(aarch64)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy 22.04(x86_64)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy 22.04(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble 24.04(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble 24.04(aarch64)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Oraclelinux 8 tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } } } stage('Sign packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { signRPM() signDEB() } } stage('Push to public repository') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { // sync packages - sync2ProdAutoBuild(PBM_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PBM_REPO, COMPONENT) } } + stage('Build docker containers for PerconaLab') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1200 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-backup-mongodb + sed -i "s/ENV PBM_VERSION.*/ENV PBM_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PBM_REPO_CH.*/ENV PBM_REPO_CH ${PBM_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-backup-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-backup-mongodb-amd64 perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 + sed -i "s/ENV PBM_VERSION.*/ENV PBM_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PBM_REPO_CH.*/ENV PBM_REPO_CH ${PBM_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-backup-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-backup-mongodb-arm64 perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 + sudo docker push perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + PBM_MAJOR_VERSION=$(echo $VERSION | cut -d'.' -f1) + PBM_MINOR_VERSION=$(echo $VERSION | cut -d'.' -f2) + PBM_PATCH_VERSION=$(echo $VERSION | cut -d'.' -f3) + sudo docker manifest create --amend perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} \ + perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 \ + perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} + + sudo docker manifest create --amend perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} \ + perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 \ + perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} + + sudo docker manifest create --amend perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION} \ + perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 \ + perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION} perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION} perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION} + + sudo docker manifest push perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} + sudo docker manifest push perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} + sudo docker manifest push perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION} + ''' + } + } + } + } + stage('Build docker containers for DockerHub registry') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'DockerHub' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1200 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-backup-mongodb + sed -i "s/ENV PBM_VERSION.*/ENV PBM_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PBM_REPO_CH.*/ENV PBM_REPO_CH ${PBM_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-backup-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-backup-mongodb-amd64 percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 + + sed -i "s/ENV PBM_VERSION.*/ENV PBM_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PBM_REPO_CH.*/ENV PBM_REPO_CH ${PBM_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-backup-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-backup-mongodb-arm64 percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 + sudo docker push percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + PBM_MAJOR_VERSION=$(echo $VERSION | cut -d'.' -f1) + PBM_MINOR_VERSION=$(echo $VERSION | cut -d'.' -f2) + PBM_PATCH_VERSION=$(echo $VERSION | cut -d'.' -f3) + sudo docker manifest create --amend percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} \ + percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 \ + percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} + + sudo docker manifest create --amend percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} \ + percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 \ + percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} + + sudo docker manifest create --amend percona/percona-backup-mongodb:${PBM_MAJOR_VERSION} \ + percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 \ + percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-backup-mongodb:${PBM_MAJOR_VERSION} percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-backup-mongodb:${PBM_MAJOR_VERSION} percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-backup-mongodb:${PBM_MAJOR_VERSION} + + sudo docker manifest push percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} + sudo docker manifest push percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} + sudo docker manifest push percona/percona-backup-mongodb:${PBM_MAJOR_VERSION} + ''' + } + } + } + } } post { success { - slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") script { currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" } diff --git a/pbm/jenkins/percona-mongodb-backup.yml b/pbm/jenkins/percona-mongodb-backup.yml index 4403b1c4d4..533d2869c3 100644 --- a/pbm/jenkins/percona-mongodb-backup.yml +++ b/pbm/jenkins/percona-mongodb-backup.yml @@ -1,5 +1,5 @@ - job: - name: pbm-autobuild-RELEASE + name: hetzner-pbm-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -9,7 +9,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pbm/jenkins/percona-mongodb-backup.groovy diff --git a/pbm/pbm-docker-arm.groovy b/pbm/pbm-docker-arm.groovy index ae8198d04e..1e06e37d22 100644 --- a/pbm/pbm-docker-arm.groovy +++ b/pbm/pbm-docker-arm.groovy @@ -1,16 +1,17 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'docker-64gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') choice(name: 'PBM_REPO_CH', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PBM_VERSION', defaultValue: '2.3.0-1', description: 'PBM version') choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') diff --git a/pbm/pbm-docker-arm.yml b/pbm/pbm-docker-arm.yml index 63f321df3a..00941cf881 100644 --- a/pbm/pbm-docker-arm.yml +++ b/pbm/pbm-docker-arm.yml @@ -1,5 +1,5 @@ - job: - name: pbm-docker-arm + name: hetzner-pbm-docker-arm project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pbm/pbm-docker-arm.groovy diff --git a/pbm/pbm-docker.groovy b/pbm/pbm-docker.groovy index 5eee84ae00..80d75bc86c 100644 --- a/pbm/pbm-docker.groovy +++ b/pbm/pbm-docker.groovy @@ -1,16 +1,17 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') choice(name: 'PBM_REPO_CH', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PBM_VERSION', defaultValue: '2.0.2-1', description: 'PBM version') choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') diff --git a/pbm/pbm-docker.yml b/pbm/pbm-docker.yml index e38b2cb68c..bca6a88408 100644 --- a/pbm/pbm-docker.yml +++ b/pbm/pbm-docker.yml @@ -1,5 +1,5 @@ - job: - name: pbm-docker + name: hetzner-pbm-docker project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pbm/pbm-docker.groovy diff --git a/pbm/pbm-e2e-tests.groovy b/pbm/pbm-e2e-tests.groovy index 6dc8c6a99e..12e32ca3ea 100644 --- a/pbm/pbm-e2e-tests.groovy +++ b/pbm/pbm-e2e-tests.groovy @@ -30,17 +30,20 @@ void prepareCluster(String TEST_TYPE) { git poll: false, branch: params.PBM_BRANCH, url: 'https://github.com/percona/percona-backup-mongodb.git' - withCredentials([file(credentialsId: 'PBM-AWS-S3', variable: 'PBM_AWS_S3_YML'), file(credentialsId: 'PBM-GCS-S3', variable: 'PBM_GCS_S3_YML'), file(credentialsId: 'PBM-AZURE', variable: 'PBM_AZURE_YML')]) { + withCredentials([file(credentialsId: 'PBM-AWS-S3', variable: 'PBM_AWS_S3_YML'), file(credentialsId: 'PBM-GCS-S3', variable: 'PBM_GCS_S3_YML'), file(credentialsId: 'PBM-GCS-HMAC-S3', variable: 'PBM_GCS_HMAC_S3_YML'), file(credentialsId: 'PBM-AZURE', variable: 'PBM_AZURE_YML')]) { sh """ cp $PBM_AWS_S3_YML ./e2e-tests/docker/conf/aws.yaml cp $PBM_GCS_S3_YML ./e2e-tests/docker/conf/gcs.yaml + cp $PBM_GCS_HMAC_S3_YML ./e2e-tests/docker/conf/gcs_hmac.yaml cp $PBM_AZURE_YML ./e2e-tests/docker/conf/azure.yaml sed -i s:pbme2etest:pbme2etest-${TEST_TYPE}:g ./e2e-tests/docker/conf/aws.yaml sed -i s:pbme2etest:pbme2etest-${TEST_TYPE}:g ./e2e-tests/docker/conf/gcs.yaml + sed -i s:pbme2etest:pbme2etest-${TEST_TYPE}:g ./e2e-tests/docker/conf/gcs_hmac.yaml sed -i s:pbme2etest:pbme2etest-${TEST_TYPE}:g ./e2e-tests/docker/conf/azure.yaml chmod 664 ./e2e-tests/docker/conf/aws.yaml chmod 664 ./e2e-tests/docker/conf/gcs.yaml + chmod 664 ./e2e-tests/docker/conf/gcs_hmac.yaml chmod 664 ./e2e-tests/docker/conf/azure.yaml @@ -49,7 +52,7 @@ void prepareCluster(String TEST_TYPE) { } } -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) @@ -57,12 +60,13 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') string(name: 'PBM_BRANCH', defaultValue: 'dev', description: 'PBM branch') } triggers { @@ -80,7 +84,7 @@ pipeline { parallel { stage('New cluster 8.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('80-newc-logic') @@ -89,7 +93,7 @@ pipeline { } stage('New cluster 6.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('60-newc-logic') @@ -98,7 +102,7 @@ pipeline { } stage('New cluster 7.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('70-newc-logic') @@ -107,7 +111,7 @@ pipeline { } stage('Sharded 8.0 logical') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { prepareCluster('80-shrd-logic') @@ -116,7 +120,7 @@ pipeline { } stage('Sharded 6.0 logical') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { prepareCluster('60-shrd-logic') @@ -125,7 +129,7 @@ pipeline { } stage('Sharded 7.0 logical') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { prepareCluster('70-shrd-logic') @@ -134,7 +138,7 @@ pipeline { } stage('Non-sharded 8.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('80-rs-logic') @@ -143,7 +147,7 @@ pipeline { } stage('Non-sharded 6.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('60-rs-logic') @@ -152,7 +156,7 @@ pipeline { } stage('Non-sharded 7.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('70-rs-logic') @@ -161,7 +165,7 @@ pipeline { } stage('Single-node 8.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('80-single-logic') @@ -170,7 +174,7 @@ pipeline { } stage('Single-node 6.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('60-single-logic') @@ -179,7 +183,7 @@ pipeline { } stage('Single-node 7.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('70-single-logic') @@ -188,7 +192,7 @@ pipeline { } stage('Sharded 8.0 physical') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { prepareCluster('80-shrd-phys') @@ -197,7 +201,7 @@ pipeline { } stage('Sharded 6.0 physical') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { prepareCluster('60-shrd-phys') @@ -206,7 +210,7 @@ pipeline { } stage('Sharded 7.0 physical') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { prepareCluster('70-shrd-phys') @@ -215,7 +219,7 @@ pipeline { } stage('Non-sharded 8.0 physical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('80-rs-phys') @@ -224,7 +228,7 @@ pipeline { } stage('Non-sharded 6.0 physical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('60-rs-phys') @@ -233,7 +237,7 @@ pipeline { } stage('Non-sharded 7.0 physical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('70-rs-phys') @@ -242,7 +246,7 @@ pipeline { } stage('Single-node 8.0 physical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('80-single-phys') @@ -251,7 +255,7 @@ pipeline { } stage('Single-node 6.0 physical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('60-single-phys') @@ -260,7 +264,7 @@ pipeline { } stage('Single-node 7.0 physical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('70-single-phys') diff --git a/pbm/pbm-e2e-tests.yml b/pbm/pbm-e2e-tests.yml index 0d62378029..da29844f9b 100644 --- a/pbm/pbm-e2e-tests.yml +++ b/pbm/pbm-e2e-tests.yml @@ -1,5 +1,5 @@ - job: - name: pbm-e2e-tests + name: hetzner-pbm-e2e-tests project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'PBM_psmdb_jenkins' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pbm/pbm-e2e-tests.groovy diff --git a/pbm/pbm-functional-tests-full.groovy b/pbm/pbm-functional-tests-full.groovy index 0affd472e8..a69f93e60a 100644 --- a/pbm/pbm-functional-tests-full.groovy +++ b/pbm/pbm-functional-tests-full.groovy @@ -12,7 +12,7 @@ pipeline { } parameters { string(name: 'PBM_BRANCH', defaultValue: 'main', description: 'PBM branch or commit') - string(name: 'GO_VER', defaultValue: '1.22-bullseye', description: 'GOLANG docker image for building PBM from sources') + string(name: 'GO_VER', defaultValue: 'bullseye', description: 'GOLANG docker image for building PBM from sources') choice(name: 'instance', choices: ['docker-64gb','docker-64gb-aarch64'], description: 'Ec2 instance type for running tests') string(name: 'TESTING_BRANCH', defaultValue: 'main', description: 'psmdb-testing repo branch') } diff --git a/pbm/pbm-functional-tests.groovy b/pbm/pbm-functional-tests.groovy index 6c10d447d4..685e316dec 100644 --- a/pbm/pbm-functional-tests.groovy +++ b/pbm/pbm-functional-tests.groovy @@ -13,7 +13,7 @@ pipeline { parameters { string(name: 'PBM_BRANCH', defaultValue: 'main', description: 'PBM branch') string(name: 'PSMDB', defaultValue: 'percona/percona-server-mongodb', description: 'PSMDB docker image') - string(name: 'GO_VER', defaultValue: '1.22-bullseye', description: 'GOLANG docker image for building PBM from sources') + string(name: 'GO_VER', defaultValue: 'bullseye', description: 'GOLANG docker image for building PBM from sources') choice(name: 'instance', choices: ['docker-64gb','docker-64gb-aarch64'], description: 'Ec2 instance type for running tests') string(name: 'TESTING_BRANCH', defaultValue: 'main', description: 'psmdb-testing repo branch') } diff --git a/pbm/pbm-release-test-run.groovy b/pbm/pbm-release-test-run.groovy index 8a424c3733..ec966ffb82 100644 --- a/pbm/pbm-release-test-run.groovy +++ b/pbm/pbm-release-test-run.groovy @@ -1,16 +1,17 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'master' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'master' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') string(name: 'PBM_VERSION', defaultValue: '2.0.5', description: 'PBM Version') string(name: 'PBM_BRANCH', defaultValue: 'release-2.0.5', description: 'PBM Branch') } @@ -20,7 +21,7 @@ pipeline { stages { stage ('Run e2e tests') { steps { - build job: 'pbm-functional-tests-full', propagate: false, wait: true, parameters: [ string(name: 'PBM_BRANCH', value: params.PBM_BRANCH) ] + build job: 'hetzner-pbm-functional-tests-full', propagate: false, wait: true, parameters: [ string(name: 'PBM_BRANCH', value: params.PBM_BRANCH) ] } } stage ('Run package tests') { @@ -32,7 +33,7 @@ pipeline { steps { script { def version = params.PBM_VERSION + '-1' - build job: 'pbm-docker', parameters: [string(name: 'PBM_REPO_CH', value: "testing"), string(name: 'PBM_VERSION', value: version ), string(name: 'LATEST', value: "no") ] + build job: 'hetzner-pbm-docker', parameters: [string(name: 'PBM_REPO_CH', value: "testing"), string(name: 'PBM_VERSION', value: version ), string(name: 'LATEST', value: "no") ] } } } diff --git a/pbm/pbm-release-test-run.yml b/pbm/pbm-release-test-run.yml index aae98ef360..b369215e88 100644 --- a/pbm/pbm-release-test-run.yml +++ b/pbm/pbm-release-test-run.yml @@ -1,5 +1,5 @@ - job: - name: pbm-release-test-run + name: hetzner-pbm-release-test-run project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pbm/pbm-release-test-run.groovy diff --git a/pdmdb/pdmdb-site-check.groovy b/pdmdb/pdmdb-site-check.groovy index 30bba1c312..cc1d0dd23c 100644 --- a/pdmdb/pdmdb-site-check.groovy +++ b/pdmdb/pdmdb-site-check.groovy @@ -5,12 +5,16 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice( + name: 'CLOUD', + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build') string( defaultValue: '6.0.9-7', description: 'PSMDB Version for tests', diff --git a/pdmdb/pdmdb-site-check.yml b/pdmdb/pdmdb-site-check.yml index 255590420a..f9697e0834 100644 --- a/pdmdb/pdmdb-site-check.yml +++ b/pdmdb/pdmdb-site-check.yml @@ -1,5 +1,5 @@ - job: - name: pdmdb-site-check + name: hetzner-pdmdb-site-check project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pdmdb/pdmdb-site-check.groovy diff --git a/pdps/perl-DBD-Mysql.groovy b/pdps/perl-DBD-Mysql.groovy index 505c8ebfe3..da80d88490 100644 --- a/pdps/perl-DBD-Mysql.groovy +++ b/pdps/perl-DBD-Mysql.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/perl5-dbi/DBD-mysql.git', description: 'URL for perl-DBD-MySQL repository', @@ -88,36 +92,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build perl-DBD-MySQL generic source packages') { parallel { stage('Build perl-DBD-MySQL generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build perl-DBD-MySQL generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -126,184 +130,184 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } diff --git a/pdps/perl-DBD-Mysql.yml b/pdps/perl-DBD-Mysql.yml index 9626d00dd4..46d1929026 100644 --- a/pdps/perl-DBD-Mysql.yml +++ b/pdps/perl-DBD-Mysql.yml @@ -1,5 +1,5 @@ - job: - name: perl-DBD-MySQL-RELEASE + name: hetzner-perl-DBD-MySQL-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pdps/perl-DBD-Mysql.groovy diff --git a/pdpxc/percona-replication-manager.groovy b/pdpxc/percona-replication-manager.groovy index 01d298c02b..7ce283e401 100644 --- a/pdpxc/percona-replication-manager.groovy +++ b/pdpxc/percona-replication-manager.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/replication-manager.git', description: 'URL for percona-replication-manager repository', @@ -66,7 +70,7 @@ pipeline { stages { stage('Create PRM source tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -85,36 +89,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PRM generic source packages') { parallel { stage('Build PRM generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PRM generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -123,197 +127,197 @@ pipeline { parallel { stage('Centos 7') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } diff --git a/pdpxc/percona-replication-manager.yml b/pdpxc/percona-replication-manager.yml index 9af712b329..6c3911ff7c 100644 --- a/pdpxc/percona-replication-manager.yml +++ b/pdpxc/percona-replication-manager.yml @@ -1,5 +1,5 @@ - job: - name: prm-autobuild-RELEASE + name: hetzner-prm-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pdpxc/percona-replication-manager.groovy diff --git a/percona-release/jenkins/percona-release.groovy b/percona-release/jenkins/percona-release.groovy index 16043cbdc6..38f6fc3efd 100644 --- a/percona-release/jenkins/percona-release.groovy +++ b/percona-release/jenkins/percona-release.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -113,9 +113,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-repositories.git', description: 'URL for mysql-shell packaging repository', @@ -161,15 +165,15 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build percona-release packages') { parallel { stage('RPM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -182,15 +186,15 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('DEB') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -202,14 +206,14 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } /* stage('RPM ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -221,13 +225,13 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('DEB ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -239,8 +243,8 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadRPMfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } */ diff --git a/percona-release/jenkins/percona-release.yml b/percona-release/jenkins/percona-release.yml index 34a1cf2d92..61257f56e8 100644 --- a/percona-release/jenkins/percona-release.yml +++ b/percona-release/jenkins/percona-release.yml @@ -1,5 +1,5 @@ - job: - name: percona-release-RELEASE + name: hetzner-percona-release-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: percona-release/jenkins/percona-release.groovy diff --git a/percona-telemetry-agent/percona-telemetry-agent.groovy b/percona-telemetry-agent/percona-telemetry-agent.groovy index 0e4c1481bf..dded936315 100644 --- a/percona-telemetry-agent/percona-telemetry-agent.groovy +++ b/percona-telemetry-agent/percona-telemetry-agent.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/telemetry-agent.git', description: 'URL for telemetry-agent repository', @@ -66,7 +70,7 @@ pipeline { stages { stage('Create telemetry agent source tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { //slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build telemetry agent generic source packages') { parallel { stage('Build telemetry agent generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build telemetry agent generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:buster", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,210 +126,210 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Amazon Linux 2023') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("amazonlinux:2023", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Amazon Linux 2023 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("amazonlinux:2023", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -333,14 +337,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PTA_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PTA_REPO, COMPONENT) } } diff --git a/percona-telemetry-agent/percona-telemetry-agent.yml b/percona-telemetry-agent/percona-telemetry-agent.yml index f30cbc4a58..3123fa8cf1 100644 --- a/percona-telemetry-agent/percona-telemetry-agent.yml +++ b/percona-telemetry-agent/percona-telemetry-agent.yml @@ -1,5 +1,5 @@ - job: - name: percona-telemetry-agent-RELEASE + name: hetzner-percona-telemetry-agent-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: percona-telemetry-agent/percona-telemetry-agent.groovy diff --git a/plm/get-plm-branches.groovy b/plm/get-plm-branches.groovy new file mode 100644 index 0000000000..42ff411cee --- /dev/null +++ b/plm/get-plm-branches.groovy @@ -0,0 +1,109 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' + } + parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') + string( + defaultValue: 'https://github.com/percona/percona-link-mongodb.git', + description: 'URL for percona-link-mongodb repository', + name: 'GIT_REPO') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Get release branches') { + steps { + script { + String S3_STASH = (params.CLOUD == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (params.CLOUD == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + EC=0 + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 ls s3://percona-jenkins-artifactory/percona-link-mongodb/branch_commit_id.properties ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 || EC=\$? + + if [ \${EC} = 1 ]; then + LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release\\* | tail -1) + BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) + COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) + + echo "BRANCH_NAME=\${BRANCH_NAME}" > branch_commit_id.properties + echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id.properties + + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp branch_commit_id.properties s3://percona-jenkins-artifactory/percona-link-mongodb/ ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 + echo "START_NEW_BUILD=NO" > startBuild + else + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp s3://percona-jenkins-artifactory/percona-link-mongodb/branch_commit_id.properties . ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 + source branch_commit_id.properties + + LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release\\* | tail -1) + LATEST_BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) + LATEST_COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) + + if [ "x\${COMMIT_ID}" != "x\${LATEST_COMMIT_ID}" ] || [ "x\${BRANCH_NAME}" != "x\${LATEST_BRANCH_NAME}" ]; then + echo "START_NEW_BUILD=YES" > startBuild + else + echo "START_NEW_BUILD=NO" > startBuild + fi + + echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id.properties + echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id.properties + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp branch_commit_id.properties s3://percona-jenkins-artifactory/percona-link-mongodb/ ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 + fi + """ + } + START_NEW_BUILD = sh(returnStdout: true, script: "source startBuild; echo \${START_NEW_BUILD}").trim() + BRANCH_NAME = sh(returnStdout: true, script: "source branch_commit_id.properties; echo \${BRANCH_NAME}").trim() + COMMIT_ID = sh(returnStdout: true, script: "source branch_commit_id.properties; echo \${COMMIT_ID}").trim() + VERSION = sh(returnStdout: true, script: "source branch_commit_id.properties; echo \${BRANCH_NAME} | sed s:release\\-::").trim() + } + } + } + stage('Build needed') { + when { + expression { START_NEW_BUILD == 'YES' } + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + echo ${START_NEW_BUILD}: build required + """ + } + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") + build job: 'hetzner-plm-autobuild-RELEASE', parameters: [string(name: 'CLOUD', value: CLOUD), string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'VERSION', value: VERSION), string(name: 'COMPONENT', value: 'testing')] + } + } + stage('Build skipped') { + when { + expression { START_NEW_BUILD == 'NO' } + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + echo ${START_NEW_BUILD} build required + """ + } + } + } + } + post { + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/plm/get-plm-branches.yml b/plm/get-plm-branches.yml new file mode 100644 index 0000000000..d63d994de9 --- /dev/null +++ b/plm/get-plm-branches.yml @@ -0,0 +1,17 @@ +- job: + name: hetzner-plm-autobuild + project-type: pipeline + description: | + Do not edit this job through the web! + Check PLM repo for new release branches to build packages + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: plm/get-plm-branches.groovy + triggers: + - timed: "H/15 * * * *" diff --git a/plm/percona-link-mongodb.groovy b/plm/percona-link-mongodb.groovy new file mode 100644 index 0000000000..fcbfdb70d8 --- /dev/null +++ b/plm/percona-link-mongodb.groovy @@ -0,0 +1,345 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/packaging/scripts/percona-link-mongodb_builder.sh -O percona-link-mongodb_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./percona-link-mongodb_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./percona-link-mongodb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${VERSION} --branch=${GIT_BRANCH} --rpm_release=${RELEASE} --deb_release=${RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/percona-link-mongodb.git', + description: 'URL for percona-link-mongodb repository', + name: 'GIT_REPO') + string( + defaultValue: 'main', + description: 'Tag/Branch for percona-link-mongodb repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM/DEB release value', + name: 'RELEASE') + string( + defaultValue: '1.0.1', + description: 'VERSION value', + name: 'VERSION') + string( + defaultValue: 'plm', + description: 'PTA repo name', + name: 'PTA_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Create Percona Link for MongoDB source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + //slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-link-mongodb.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/percona-link-mongodb.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build Percona Link for MongoDB generic source packages') { + parallel { + stage('Build Percona Link for MongoDB generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build Percona Link for MongoDB generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("debian:buster", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build Percona Link for MongoDB RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Amazon Linux 2023') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Amazon Linux 2023 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PTA_REPO, COMPONENT) + } + } + + } + post { + success { + //slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" + } + deleteDir() + } + failure { + //slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/plm/percona-link-mongodb.yml b/plm/percona-link-mongodb.yml new file mode 100644 index 0000000000..3c8f70b8ee --- /dev/null +++ b/plm/percona-link-mongodb.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-plm-autobuild-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: plm/percona-link-mongodb.groovy + diff --git a/plm/plm-functional-tests.groovy b/plm/plm-functional-tests.groovy new file mode 100644 index 0000000000..bed2f67e30 --- /dev/null +++ b/plm/plm-functional-tests.groovy @@ -0,0 +1,120 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +def set_agent(cloud, arch) { + if (arch == 'x86') { + if (cloud == 'Hetzner') { + return 'docker-x64' + } else { + return 'docker-64gb' + } + } else if (arch == 'arm') { + if (cloud == 'Hetzner') { + return 'docker-aarch64' + } else { + return 'docker-64gb-aarch64' + } + } +} + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'master' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') + string(name: 'PLM_BRANCH', defaultValue: 'main', description: 'PLM branch') + string(name: 'GO_VER', defaultValue: 'latest', description: 'GOLANG docker image for building PBM from sources') + choice(name: 'ARCH', choices: ['x86','arm'], description: 'Ec2 instance type for running tests') + string(name: 'PSMDB_TESTING_BRANCH', defaultValue: 'main', description: 'psmdb-testing repo branch') + string(name: 'TEST_FILTER', defaultValue: '', description: 'Optional pytest filter, f.e. T2 or T3') + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PLM_BRANCH}" + } + } + } + stage ('Run tests') { + matrix { + agent { + label set_agent(params.CLOUD, params.ARCH) + } + axes { + axis { + name 'MONGODB_IMAGE' + values 'percona/percona-server-mongodb:6.0', 'percona/percona-server-mongodb:7.0', 'percona/percona-server-mongodb:8.0' + } + } + stages { + stage ('Run tests') { + steps { + withCredentials([string(credentialsId: 'olexandr_zephyr_token', variable: 'ZEPHYR_TOKEN')]) { + sh """ + docker kill \$(docker ps -a -q) || true + docker rm \$(docker ps -a -q) || true + docker rmi -f \$(docker images -q | uniq) || true + sudo rm -rf ./* + if [ ! -f "/usr/local/bin/docker-compose" ] ; then + if [ ${params.INSTANCE} = "docker-64gb-aarch64" ]; then + sudo curl -SL https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-linux-aarch64 -o /usr/local/bin/docker-compose + else + sudo curl -SL https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose + fi + sudo chmod +x /usr/local/bin/docker-compose + fi + """ + + dir('psmdb-testing') { + git poll: false, branch: params.PSMDB_TESTING_BRANCH, url: 'https://github.com/Percona-QA/psmdb-testing.git' + } + + sh """ + cd psmdb-testing/plm-pytest + docker-compose build --no-cache + docker-compose up -d + if [ -n "${params.TEST_FILTER}" ]; then + docker-compose run test pytest -v -s -k "${params.TEST_FILTER}" --junitxml=junit.xml || true + else + docker-compose run test pytest -v -s --junitxml=junit.xml || true + fi + docker-compose down -v --remove-orphans + curl -H "Content-Type:multipart/form-data" -H "Authorization: Bearer ${ZEPHYR_TOKEN}" -F "file=@junit.xml;type=application/xml" 'https://api.zephyrscale.smartbear.com/v2/automations/executions/junit?projectKey=PLM' -F 'testCycle={"name":"${JOB_NAME}-${BUILD_NUMBER}","customFields": { "PLM branch": "${PLM_BRANCH}","PSMDB docker image": "${MONGODB_IMAGE}","Instance": "${params.INSTANCE}"}};type=application/json' -i || true + """ + } + } + post { + always { + junit testResults: "**/junit.xml", keepLongStdio: true, allowEmptyResults: true, skipPublishingChecks: true + sh """ + docker kill \$(docker ps -a -q) || true + docker rm \$(docker ps -a -q) || true + docker rmi -f \$(docker images -q | uniq) || true + sudo rm -rf ./* + """ + } + } + } + } + } + } + } + post { + success { + slackNotify("#mongodb_autofeed", "#00FF00", "[${JOB_NAME}]: PLM ${PLM_BRANCH} - all tests passed") + } + unstable { + slackNotify("#mongodb_autofeed", "#F6F930", "[${JOB_NAME}]: PLM ${PLM_BRANCH} - some tests failed [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#mongodb_autofeed", "#FF0000", "[${JOB_NAME}]: PLM ${PLM_BRANCH} - unexpected failure [${BUILD_URL}]") + } + } +} diff --git a/plm/plm-functional-tests.yml b/plm/plm-functional-tests.yml new file mode 100644 index 0000000000..bd1023ffeb --- /dev/null +++ b/plm/plm-functional-tests.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-plm-functional-tests + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: plm/plm-functional-tests.groovy diff --git a/pmm/v3/pmm3-migration-tests.groovy b/pmm/v3/pmm3-migration-tests.groovy new file mode 100644 index 0000000000..716decd9e1 --- /dev/null +++ b/pmm/v3/pmm3-migration-tests.groovy @@ -0,0 +1,357 @@ +library changelog: false, identifier: 'lib@master', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +def versionsList = pmmVersion('list').reverse() +def latestVersion = versionsList.first() +def getMinorVersion(VERSION) { + return VERSION.split("\\.")[1].toInteger() +} + +pipeline { + agent { + label 'agent-amd64-ol9' + } + environment { + REMOTE_AWS_MYSQL_USER=credentials('pmm-dev-mysql-remote-user') + REMOTE_AWS_MYSQL_PASSWORD=credentials('pmm-dev-remote-password') + REMOTE_AWS_MYSQL57_HOST=credentials('pmm-dev-mysql57-remote-host') + REMOTE_MYSQL_HOST=credentials('mysql-remote-host') + REMOTE_MYSQL_USER=credentials('mysql-remote-user') + REMOTE_MYSQL_PASSWORD=credentials('mysql-remote-password') + GCP_SERVER_IP=credentials('GCP_SERVER_IP') + GCP_USER=credentials('GCP_USER') + GCP_USER_PASSWORD=credentials('GCP_USER_PASSWORD') + REMOTE_MONGODB_HOST=credentials('qa-remote-mongodb-host') + REMOTE_MONGODB_USER=credentials('qa-remote-mongodb-user') + REMOTE_MONGODB_PASSWORD=credentials('qa-remote-mongodb-password') + REMOTE_POSTGRESQL_HOST=credentials('qa-remote-pgsql-host') + REMOTE_POSTGRESQL_USER=credentials('qa-remote-pgsql-user') + REMOTE_POSTGRESSQL_PASSWORD=credentials('qa-remote-pgsql-password') + REMOTE_PROXYSQL_HOST=credentials('qa-remote-proxysql-host') + REMOTE_PROXYSQL_USER=credentials('qa-remote-proxysql-user') + REMOTE_PROXYSQL_PASSWORD=credentials('qa-remote-proxysql-password') + INFLUXDB_ADMIN_USER=credentials('influxdb-admin-user') + INFLUXDB_ADMIN_PASSWORD=credentials('influxdb-admin-password') + INFLUXDB_USER=credentials('influxdb-user') + INFLUXDB_USER_PASSWORD=credentials('influxdb-user-password') + MONITORING_HOST=credentials('monitoring-host') + PMM_QA_AURORA2_MYSQL_HOST=credentials('PMM_QA_AURORA2_MYSQL_HOST') + PMM_QA_AURORA2_MYSQL_PASSWORD=credentials('PMM_QA_AURORA2_MYSQL_PASSWORD') + PMM_QA_AWS_ACCESS_KEY_ID=credentials('PMM_QA_AWS_ACCESS_KEY_ID') + PMM_QA_AWS_ACCESS_KEY=credentials('PMM_QA_AWS_ACCESS_KEY') + MAILOSAUR_API_KEY=credentials('MAILOSAUR_API_KEY') + MAILOSAUR_SERVER_ID=credentials('MAILOSAUR_SERVER_ID') + MAILOSAUR_SMTP_PASSWORD=credentials('MAILOSAUR_SMTP_PASSWORD') + ZEPHYR_PMM_API_KEY=credentials('ZEPHYR_PMM_API_KEY') + } + parameters { + string( + defaultValue: 'v3', + description: 'Tag/Branch for UI Tests repository PMM V3', + name: 'PMM_V3_UI_GIT_BRANCH') + string( + defaultValue: 'main', + description: 'Tag/Branch for UI Tests repository PMM V2', + name: 'PMM_V2_UI_GIT_BRANCH') + string( + defaultValue: latestVersion, + description: 'PMM Server Version to test for Upgrade', + name: 'DOCKER_VERSION') + string( + defaultValue: latestVersion, + description: 'PMM Client Version to test for Upgrade', + name: 'CLIENT_VERSION') + string( + defaultValue: 'pmm3admin!', + description: 'pmm-server admin user default password', + name: 'ADMIN_PASSWORD') + string( + defaultValue: 'main', + description: 'Tag/Branch for pmm-qa repository', + name: 'PMM_QA_GIT_BRANCH') + choice( + choices: ['experimental', 'testing', 'release'], + description: 'Select pmm repo to upgrate to', + name: 'UPGRADE_TAG') + } + options { + skipDefaultCheckout() + } + stages { + stage('Prepare') { + steps { + // fetch pmm-ui-tests repository + git poll: false, + branch: PMM_V3_UI_GIT_BRANCH, + url: 'https://github.com/percona/pmm-ui-tests.git' + + slackSend channel: '#pmm-notifications', color: '#0000FF', message: "[${JOB_NAME}]: build started - ${BUILD_URL}" + sh ''' + sudo mkdir -p /srv/pmm-qa || : + pushd /srv/pmm-qa + sudo git clone --single-branch --branch ${PMM_QA_GIT_BRANCH} https://github.com/percona/pmm-qa.git . + sudo git checkout ${PMM_QA_GIT_COMMIT_HASH} + popd + sudo ln -s /usr/bin/chromium-browser /usr/bin/chromium + ''' + } + } + stage('Start Server Instance') { + steps { + sh ''' + docker network create pmm-qa || true + git checkout ${PMM_V2_UI_GIT_BRANCH} + PWD=$(pwd) PMM_SERVER_IMAGE=percona/pmm-server:${DOCKER_VERSION} docker-compose up -d + + ''' + waitForContainer('pmm-server', 'pmm-managed entered RUNNING state') + waitForContainer('pmm-agent_mongo', 'waiting for connections on port 27017') + waitForContainer('pmm-agent_mysql_5_7', "Server hostname (bind-address):") + waitForContainer('pmm-agent_postgres', 'PostgreSQL init process complete; ready for start up.') + sleep 20 + sh """ + bash -x testdata/db_setup.sh + """ + script { + env.SERVER_IP = "127.0.0.1" + env.PMM_UI_URL = "http://${env.SERVER_IP}/" + env.PMM_URL = "http://admin:${env.ADMIN_PASSWORD}@${env.SERVER_IP}" + } + } + } + stage('Change admin password') { + when { + expression { getMinorVersion(DOCKER_VERSION) >= 27 } + } + steps { + sh ''' + docker exec pmm-server change-admin-password ${ADMIN_PASSWORD} + ''' + } + } + stage('Enable Repo') { + parallel { + stage('Enable Experimental Repo') { + when { + expression { env.UPGRADE_TAG == "experimental" } + } + steps { + script { + sh """ + set -o errexit + set -o xtrace + docker exec pmm-server sed -i'' -e 's^/release/^/experimental/^' /etc/yum.repos.d/pmm2-server.repo + docker exec pmm-server percona-release enable pmm2-client experimental + docker exec pmm-server yum clean all + docker exec pmm-server yum clean metadata + """ + } + } + } + stage('Enable Testing Repo') { + when { + expression { env.UPGRADE_TAG == "testing" } + } + steps { + script { + sh """ + set -o errexit + set -o xtrace + docker exec pmm-server sed -i'' -e 's^/release/^/testing/^' /etc/yum.repos.d/pmm2-server.repo + docker exec pmm-server percona-release enable pmm2-client testing + docker exec pmm-server yum clean all + docker exec pmm-server yum clean metadata + """ + } + } + } + stage('Enable Release Repo') { + when { + expression { env.UPGRADE_TAG == "release" } + } + steps { + script { + sh """ + set -o errexit + set -o xtrace + docker exec pmm-server yum clean all + docker exec pmm-server yum clean metadata + """ + } + } + } + } + } + stage('Setup Client for PMM-Server') { + steps { + script{ + if(env.UPGRADE_TAG == "experimental") { + setupPMMClient(SERVER_IP, CLIENT_VERSION.trim(), "pmm2", "no", "no", "no", 'compose_setup', ADMIN_PASSWORD) + } else if (env.UPGRADE_TAG == "testing") { + setupPMMClient(SERVER_IP, CLIENT_VERSION.trim(), "pmm2", "no", "yes", "no", 'compose_setup', ADMIN_PASSWORD, "no") + } else { + setupPMMClient(SERVER_IP, CLIENT_VERSION.trim(), "pmm2", "no", "no", "no", 'compose_setup', ADMIN_PASSWORD, "no") + } + } + sh """ + set -o errexit + set -o xtrace + export PATH=$PATH:/usr/sbin + export PMM_CLIENT_VERSION=${CLIENT_VERSION} + bash /srv/pmm-qa/pmm-tests/pmm-framework.sh \ + --download \ + --pdpgsql-version 17 --ps-version 8.0 --mo-version 8.0 --addclient=pdpgsql,1 --addclient=ps,1 --mongo-replica-for-backup \ + --pmm2 + sleep 20 + """ + } + } + stage('Sanity check') { + steps { + sh ''' + echo \${PMM_URL} + timeout 100 bash -c \'while [[ "$(curl -s -o /dev/null -w \'\'%{http_code}\'\' \${PMM_URL}/ping)" != "200" ]]; do sleep 5; done\' || false + ''' + } + } + stage('Prepare nightly tests on migrated pmm.') { + steps { + script { + sh """ + curl -sL https://rpm.nodesource.com/setup_20.x | sudo bash - + sudo yum install -y nodejs + node --version + npm ci + npx playwright install + envsubst < env.list > env.generated.list + """ + } + } + } + stage('Run pre migration Tests') { + options { + timeout(time: 150, unit: "MINUTES") + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'PMM_AWS_DEV', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + sed -i 's+http://localhost/+${PMM_UI_URL}/+g' pr.codecept.js + export PWD=\$(pwd); + npx codeceptjs run --reporter mocha-multi -c pr.codecept.js --grep '@pmm-pre-migration' + git checkout ${PMM_V3_UI_GIT_BRANCH} + """ + } + } + } + stage('Migrate pmm2 to pmm3') { + steps { + script { + sh """ + echo "\$UPGRADE_TAG" + if [[ "\$UPGRADE_TAG" == "experimental" ]]; then + export PERCONA_REPOSITORY="experimental" + export DOCKER_TAG="3-dev-latest" + export DOCKER_REPO="perconalab/pmm-server" + elif [[ "\$UPGRADE_TAG" == "testing" ]]; then + export DOCKER_TAG=\$(wget -q "https://registry.hub.docker.com/v2/repositories/perconalab/pmm-server/tags?page_size=25&name=rc" -O - | jq -r .results[].name | grep 3.*.*-rc\$ | sort -V | tail -n1) + export DOCKER_REPO="perconalab/pmm-server" + elif [[ "\$UPGRADE_TAG" == "release" ]]; then + export DOCKER_TAG=\$(wget -q "https://registry.hub.docker.com/v2/repositories/percona/pmm-server/tags?page_size=25" -O - | jq -r .results[].name | grep 3.*.* | sort -V | tail -n1) + export DOCKER_REPO="percona/pmm-server" + fi + + echo "Percona repository is: \$PERCONA_REPOSITORY" + echo "Docker tag is: \$DOCKER_TAG" + + wget https://raw.githubusercontent.com/percona/pmm/refs/heads/v3/get-pmm.sh + chmod +x get-pmm.sh + ./get-pmm.sh -n pmm-server -b --network-name pmm-qa --tag "\$DOCKER_TAG" --repo "\$DOCKER_REPO" + + sudo percona-release enable pmm3-client \$UPGRADE_TAG + sudo yum install -y pmm-client + + listVar="rs101 rs102 rs103 rs201 rs202 rs203" + + for i in \$listVar; do + echo "\$i" + docker exec \$i percona-release enable pmm3-client \$UPGRADE_TAG + docker exec \$i yum install -y pmm-client + docker exec \$i sed -i "s/443/8443/g" /usr/local/percona/pmm/config/pmm-agent.yaml + docker exec \$i cat /usr/local/percona/pmm/config/pmm-agent.yaml + docker exec \$i systemctl restart pmm-agent + done + """ + env.SERVER_IP = "127.0.0.1" + env.PMM_UI_URL = "https://${env.SERVER_IP}/" + env.PMM_URL = "https://admin:${env.ADMIN_PASSWORD}@${env.SERVER_IP}" + } + } + } + stage('Sleep') { + steps { + sleep 300 + } + } + stage('Run Tests') { + options { + timeout(time: 150, unit: "MINUTES") + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'PMM_AWS_DEV', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + sed -i 's+http://localhost/+${PMM_UI_URL}/+g' pr.codecept.js + export PWD=\$(pwd); + npx codeceptjs run --reporter mocha-multi -c pr.codecept.js --grep '@pmm-migration' + """ + } + } + } + } + post { + always { + sh ''' + # fetch all the logs from PMM server + curl --insecure ${PMM_URL}/logs.zip --output logs.zip || true + + # get logs from systemd pmm-agent.service + if [[ ${CLIENT_VERSION} != http* ]]; then + journalctl -u pmm-agent.service > ./pmm-agent.log + fi + + # get logs from managed and update-perform + echo --- pmm-managed logs from pmm-server --- >> pmm-managed-full.log + docker exec pmm-server cat /srv/logs/pmm-managed.log >> pmm-managed-full.log || true + docker exec pmm-server cat /srv/logs/pmm-update-perform.log >> pmm-update-perform.log || true + echo --- pmm-update-perform logs from pmm-server --- >> pmm-update-perform.log + docker cp pmm-server:/srv/logs srv-logs + tar -zcvf srv-logs.tar.gz srv-logs + + # stop the containers + docker-compose down || true + docker rm -f $(sudo docker ps -a -q) || true + docker volume rm $(sudo docker volume ls -q) || true + sudo chown -R ec2-user:ec2-user . || true + ''' + script { + archiveArtifacts artifacts: 'pmm-managed-full.log' + archiveArtifacts artifacts: 'pmm-update-perform.log' + archiveArtifacts artifacts: 'pmm-agent.log' + archiveArtifacts artifacts: 'logs.zip' + archiveArtifacts artifacts: 'srv-logs.tar.gz' + + if (currentBuild.result == null || currentBuild.result == 'SUCCESS') { + junit 'tests/output/*.xml' + slackSend botUser: true, channel: '#pmm-notifications', color: '#00FF00', message: "[${JOB_NAME}]: build finished - ${BUILD_URL}" + archiveArtifacts artifacts: 'logs.zip' + } else { + junit 'tests/output/*.xml' + slackSend botUser: true, channel: '#pmm-notifications', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result} - ${BUILD_URL}" + archiveArtifacts artifacts: 'logs.zip' + archiveArtifacts artifacts: 'tests/output/*.png' + } + } + } + } +} diff --git a/ppg/etcd-arm.groovy b/ppg/etcd-arm.groovy new file mode 100644 index 0000000000..48e397fd0e --- /dev/null +++ b/ppg/etcd-arm.groovy @@ -0,0 +1,257 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/etcd/etcd_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --pg_version=${GIT_BRANCH} --etcd_version=${ETCD_VERSION} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for etcd repository', + name: 'GIT_REPO') + string( + defaultValue: '3.5.13', + description: 'etcd version', + name: 'ETCD_VERSION') + string( + defaultValue: '16.2', + description: 'Tag/Branch for etcd packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.2', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create etcd source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/etcd.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/etcd.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build etcd generic source packages') { + parallel { + stage('Build etcd generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build etcd generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build etcd RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/etcd-arm.yml b/ppg/etcd-arm.yml new file mode 100644 index 0000000000..1afefa456a --- /dev/null +++ b/ppg/etcd-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-etcd-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/etcd-arm.groovy + diff --git a/ppg/etcd.groovy b/ppg/etcd.groovy index c2a222dd64..971badfd7a 100644 --- a/ppg/etcd.groovy +++ b/ppg/etcd.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for etcd repository', @@ -84,146 +88,237 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build etcd generic source packages') { parallel { stage('Build etcd generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build etcd generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build etcd RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + //stage('Oracle Linux 8 ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + // buildStage("oraclelinux:8", "--build_rpm=1") + + // pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + // uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + // } + //} + stage('Oracle Linux 9 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + //stage('Oracle Linux 9 ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + // buildStage("oraclelinux:9", "--build_rpm=1") + + // pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + // uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + // } + //} + stage('Ubuntu Focal(20.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + //stage('Ubuntu Focal(20.04) ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + // buildStage("ubuntu:focal", "--build_deb=1") + + // pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + // uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + // } + //} + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + //stage('Ubuntu Jammy(22.04) ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + // buildStage("ubuntu:jammy", "--build_deb=1") + + // pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + // uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + // } + //} + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + //stage('Ubuntu Noble(24.04) ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + // buildStage("ubuntu:noble", "--build_deb=1") + + // pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + // uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + // } + //} + stage('Debian Bullseye(11) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + //stage('Debian Bullseye(11) ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + // buildStage("debian:bullseye", "--build_deb=1") + + // pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + // uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + // } + //} + stage('Debian bookworm(12) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } + //stage('Debian bookworm(12) ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + // buildStage("debian:bookworm", "--build_deb=1") + + // pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + // uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + // } + //} } } stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/etcd.yml b/ppg/etcd.yml index 6ac3ccc49c..d867dc8b11 100644 --- a/ppg/etcd.yml +++ b/ppg/etcd.yml @@ -1,5 +1,5 @@ - job: - name: etcd-RELEASE + name: hetzner-etcd-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/etcd.groovy diff --git a/ppg/haproxy-arm.groovy b/ppg/haproxy-arm.groovy new file mode 100644 index 0000000000..0a52ca5dd6 --- /dev/null +++ b/ppg/haproxy-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/haproxy-builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --build_branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/haproxy-packaging.git', + description: 'URL for haproxy repository', + name: 'GIT_REPO') + string( + defaultValue: 'v2.6.14', + description: 'Tag/Branch for haproxy repository', + name: 'PG_BRANCH') + string( + defaultValue: 'main', + description: 'Tag/Branch for haproxy packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create haproxy source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/haproxy.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/haproxy.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build haproxy generic source packages') { + parallel { + stage('Build haproxy generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build haproxy generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build haproxy RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/haproxy-arm.yml b/ppg/haproxy-arm.yml new file mode 100644 index 0000000000..9b50aa6372 --- /dev/null +++ b/ppg/haproxy-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-haproxy-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/haproxy-arm.groovy diff --git a/ppg/haproxy.groovy b/ppg/haproxy.groovy index cc98a2b312..c960cb1de6 100644 --- a/ppg/haproxy.groovy +++ b/ppg/haproxy.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/haproxy-packaging.git', description: 'URL for haproxy repository', @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build haproxy generic source packages') { parallel { stage('Build haproxy generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build haproxy generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,184 +126,184 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -307,14 +311,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/haproxy.yml b/ppg/haproxy.yml index db6a0de85a..2130ec0a11 100644 --- a/ppg/haproxy.yml +++ b/ppg/haproxy.yml @@ -1,5 +1,5 @@ - job: - name: haproxy-RELEASE + name: hetzner-haproxy-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/haproxy.groovy diff --git a/ppg/llvm-arm.groovy b/ppg/llvm-arm.groovy new file mode 100644 index 0000000000..140e9f835a --- /dev/null +++ b/ppg/llvm-arm.groovy @@ -0,0 +1,155 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir -p test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/llvm/llvm_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --rpm_release=${RPM_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for llvm repository', + name: 'GIT_REPO') + string( + defaultValue: '16.3', + description: 'Tag/Branch for postgresql packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: 'ppg-16.3', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Build LLVM RPMs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + buildStage("oraclelinux:8", "--get_src_rpm=1") + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/llvm.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/llvm.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + buildStage("oraclelinux:8", "--build_rpm=1") + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + buildStage("oraclelinux:9", "--get_src_rpm=1") + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/llvm.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/llvm.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + buildStage("oraclelinux:9", "--build_rpm=1") + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/llvm-arm.yml b/ppg/llvm-arm.yml new file mode 100644 index 0000000000..34b53e66a7 --- /dev/null +++ b/ppg/llvm-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-llvm-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/llvm-arm.groovy + diff --git a/ppg/llvm.groovy b/ppg/llvm.groovy index 724bbded75..c0242bc846 100644 --- a/ppg/llvm.groovy +++ b/ppg/llvm.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for llvm repository', @@ -63,13 +67,13 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() buildStage("oraclelinux:8", "--get_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - popArtifactFolder("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/llvm.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -83,19 +87,19 @@ pipeline { } stash includes: 'uploadPath', name: 'uploadPath' buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() buildStage("oraclelinux:9", "--get_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - popArtifactFolder("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/llvm.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -109,8 +113,8 @@ pipeline { } stash includes: 'uploadPath', name: 'uploadPath' buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } @@ -118,13 +122,13 @@ pipeline { stage('Sign packages') { steps { - signRPM() + signRPM(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/llvm.yml b/ppg/llvm.yml index 367225bf20..b01532d210 100644 --- a/ppg/llvm.yml +++ b/ppg/llvm.yml @@ -1,5 +1,5 @@ - job: - name: llvm-RELEASE + name: hetzner-llvm-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/llvm.groovy diff --git a/ppg/patroni-arm.groovy b/ppg/patroni-arm.groovy new file mode 100644 index 0000000000..b2a8a23f26 --- /dev/null +++ b/ppg/patroni-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/patroni/patroni_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for patroni repository', + name: 'GIT_REPO') + string( + defaultValue: '3.1.0', + description: 'Tag/Branch for postgresql', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for patroni repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PATRONI source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/patroni.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/patroni.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PATRONI generic source packages') { + parallel { + stage('Build PATRONI generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PATRONI generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PATRONI RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/patroni-arm.yml b/ppg/patroni-arm.yml new file mode 100644 index 0000000000..f887959734 --- /dev/null +++ b/ppg/patroni-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-patroni-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/patroni-arm.groovy + diff --git a/ppg/patroni.groovy b/ppg/patroni.groovy index cc2a44385c..ee57175c56 100644 --- a/ppg/patroni.groovy +++ b/ppg/patroni.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for patroni repository', @@ -84,131 +88,157 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PATRONI generic source packages') { parallel { stage('Build PATRONI generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PATRONI generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PATRONI RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +246,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/patroni.yml b/ppg/patroni.yml index 89c355e8b3..84fd41cf85 100644 --- a/ppg/patroni.yml +++ b/ppg/patroni.yml @@ -1,5 +1,5 @@ - job: - name: patroni-RELEASE + name: hetzner-patroni-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/patroni.groovy diff --git a/ppg/percona-postgis-arm.groovy b/ppg/percona-postgis-arm.groovy new file mode 100644 index 0000000000..9a7fd34086 --- /dev/null +++ b/ppg/percona-postgis-arm.groovy @@ -0,0 +1,261 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/postgis/postgis_builder.sh -O postgis_builder.sh + pwd -P + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./postgis_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./postgis_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --postgis_gitrepo=${POSTGIS_GITREPO} --postgis_branch=${POSTGIS_BRANCH} --postgis_ver=${POSTGIS_VERSION} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for postgis packaging repository', + name: 'GIT_REPO') + string( + defaultValue: '15.4', + description: 'Tag/Branch for postgis packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: 'https://github.com/postgis/postgis.git', + description: 'URL for postgis repository', + name: 'POSTGIS_GITREPO') + string( + defaultValue: 'stable-3.3', + description: 'Tag/Branch for postgis repository', + name: 'POSTGIS_BRANCH') + string( + defaultValue: '3.3', + description: 'POSTGIS release value', + name: 'POSTGIS_VERSION') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-15.4', + description: 'POSTGIS repo name', + name: 'POSTGIS_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create POSTGIS source tarball') { + steps { + slackNotify("", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-postgis.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/percona-postgis.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build POSTGIS generic source packages') { + parallel { + stage('Build POSTGIS generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build POSTGIS generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build POSTGIS RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, POSTGIS_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/percona-postgis-arm.yml b/ppg/percona-postgis-arm.yml new file mode 100644 index 0000000000..4ea7523458 --- /dev/null +++ b/ppg/percona-postgis-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-percona-postgis-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/percona-postgis-arm.groovy + diff --git a/ppg/percona-postgis.groovy b/ppg/percona-postgis.groovy index 97d750e3ec..d537e4a767 100644 --- a/ppg/percona-postgis.groovy +++ b/ppg/percona-postgis.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -28,9 +28,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for postgis packaging repository', @@ -91,131 +95,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build POSTGIS generic source packages') { parallel { stage('Build POSTGIS generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build POSTGIS generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build POSTGIS RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } - } - stage('Ubuntu Focal(20.04)') { + } + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bookworm(12)') { + stage('Debian Bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -223,14 +318,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(POSTGIS_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, POSTGIS_REPO, COMPONENT) } } diff --git a/ppg/percona-postgis.yml b/ppg/percona-postgis.yml index 398e3d1bdc..9e8c673c14 100644 --- a/ppg/percona-postgis.yml +++ b/ppg/percona-postgis.yml @@ -1,5 +1,5 @@ - job: - name: percona-postgis-RELEASE + name: hetzner-percona-postgis-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/percona-postgis.groovy diff --git a/ppg/pg17_autobuild.groovy b/ppg/pg17_autobuild.groovy new file mode 100644 index 0000000000..fa3dc01742 --- /dev/null +++ b/ppg/pg17_autobuild.groovy @@ -0,0 +1,110 @@ +library changelog: false, identifier: 'lib@master', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +pipeline { + agent { + label 'docker' + } + parameters { + string( + defaultValue: 'https://github.com/percona/postgres.git', + description: 'URL for pg repository', + name: 'GIT_REPO') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Get release branches') { + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + EC=0 + aws s3 ls s3://percona-jenkins-artifactory/pg17/branch_commit_id_17.properties || EC=\$? + + if [ \${EC} = 1 ]; then + LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-17\\* | tail -1) + BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) + COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) + + echo "BRANCH_NAME=\${BRANCH_NAME}" > branch_commit_id_17.properties + echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id_17.properties + + aws s3 cp branch_commit_id_17.properties s3://percona-jenkins-artifactory/pg17/ + echo "START_NEW_BUILD=NO" > startBuild + else + aws s3 cp s3://percona-jenkins-artifactory/pg17/branch_commit_id_17.properties . + source ./branch_commit_id_17.properties + cat ./branch_commit_id_17.properties + + LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-17\\* | tail -1) + LATEST_BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) + LATEST_COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) + VERSION=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d - -f 2-3) + + if [ "x\${COMMIT_ID}" != "x\${LATEST_COMMIT_ID}" ] || [ "x\${BRANCH_NAME}" != "x\${LATEST_BRANCH_NAME}" ]; then + echo "START_NEW_BUILD=YES" > startBuild + else + echo "START_NEW_BUILD=NO" > startBuild + fi + + echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id_17.properties + echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id_17.properties + echo "VERSION=\${VERSION}" >> branch_commit_id_17.properties + aws s3 cp branch_commit_id_17.properties s3://percona-jenkins-artifactory/pg17/ + fi + """ + } + script { + START_NEW_BUILD = sh(returnStdout: true, script: "source ./startBuild; echo \${START_NEW_BUILD}").trim() + BRANCH_NAME = sh(returnStdout: true, script: "source ./branch_commit_id_17.properties; echo \${BRANCH_NAME}").trim() + COMMIT_ID = sh(returnStdout: true, script: "source ./branch_commit_id_17.properties; echo \${COMMIT_ID}").trim() + VERSION = sh(returnStdout: true, script: "source ./branch_commit_id_17.properties; echo \${BRANCH_NAME} | cut -d - -f 2-3 ").trim() + } + + } + + } + stage('Build packages needed') { + when { + expression { START_NEW_BUILD == 'YES' } + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + echo ${START_NEW_BUILD}: build required + """ + } + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") + build job: 'hetzner-postgresql-server-17_nightly' + build job: 'hetzner-pg_tarballs-nightly' + + } + } + stage('Build skipped') { + when { + expression { START_NEW_BUILD == 'NO' } + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + echo ${START_NEW_BUILD} build required + """ + } + } + } + } + post { + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} \ No newline at end of file diff --git a/ppg/pg17_autobuild.yml b/ppg/pg17_autobuild.yml new file mode 100644 index 0000000000..3d9ab10cb3 --- /dev/null +++ b/ppg/pg17_autobuild.yml @@ -0,0 +1,17 @@ +- job: + name: pg17-autobuild + project-type: pipeline + description: | + Do not edit this job through the web! + Check PG repo for new release branches to build packages + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pg/pg17_autobuild.groovy + triggers: + - timed: "H/15 * * * *" diff --git a/ppg/pg_cron-arm.groovy b/ppg/pg_cron-arm.groovy new file mode 100644 index 0000000000..c4b3de5ed6 --- /dev/null +++ b/ppg/pg_cron-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_cron/pg_cron_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_cron repository', + name: 'GIT_REPO') + string( + defaultValue: '16.1', + description: 'Tag/Branch for pg_cron repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.1', + description: 'Tag/Branch for pg_cron packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create pg_cron source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg_cron.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg_cron.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build pg_cron generic source packages') { + parallel { + stage('Build pg_cron generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build pg_cron generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build pg_cron RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_cron-arm.yml b/ppg/pg_cron-arm.yml new file mode 100644 index 0000000000..e3ee342076 --- /dev/null +++ b/ppg/pg_cron-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pg_cron-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_cron-arm.groovy + diff --git a/ppg/pg_cron.groovy b/ppg/pg_cron.groovy index ee050a9606..d35355eb7b 100644 --- a/ppg/pg_cron.groovy +++ b/ppg/pg_cron.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_cron repository', @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build pg_cron generic source packages') { parallel { stage('Build pg_cron generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build pg_cron generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,93 +126,93 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +220,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pg_cron.yml b/ppg/pg_cron.yml index 9b2100ee73..3082d35395 100644 --- a/ppg/pg_cron.yml +++ b/ppg/pg_cron.yml @@ -1,5 +1,5 @@ - job: - name: pg_cron-RELEASE + name: hetzner-pg_cron-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_cron.groovy diff --git a/ppg/pg_gather-arm.groovy b/ppg/pg_gather-arm.groovy new file mode 100644 index 0000000000..e59596bb74 --- /dev/null +++ b/ppg/pg_gather-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_gather/pggather_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_gather repository', + name: 'GIT_REPO') + string( + defaultValue: 'ver_1.4.4', + description: 'Tag/Branch for pg_gather repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pg_gather packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create pg_gather source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg_gather.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg_gather.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build pg_gather generic source packages') { + parallel { + stage('Build pg_gather generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build pg_gather generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build pg_gather RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_gather-arm.yml b/ppg/pg_gather-arm.yml new file mode 100644 index 0000000000..db7739ea75 --- /dev/null +++ b/ppg/pg_gather-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pg_gather-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_gather-arm.groovy + diff --git a/ppg/pg_gather.groovy b/ppg/pg_gather.groovy index eebe5d5392..8cd1093be4 100644 --- a/ppg/pg_gather.groovy +++ b/ppg/pg_gather.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_gather repository', @@ -84,131 +88,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build pg_gather generic source packages') { parallel { stage('Build pg_gather generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build pg_gather generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build pg_gather RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +311,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pg_gather.yml b/ppg/pg_gather.yml index 8000fc283f..f99ca8b797 100644 --- a/ppg/pg_gather.yml +++ b/ppg/pg_gather.yml @@ -1,5 +1,5 @@ - job: - name: pg_gather-RELEASE + name: hetzner-pg_gather-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_gather.groovy diff --git a/ppg/pg_generate_sbom.groovy b/ppg/pg_generate_sbom.groovy new file mode 100644 index 0000000000..38decca78d --- /dev/null +++ b/ppg/pg_generate_sbom.groovy @@ -0,0 +1,412 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_sbom/pg_generate_sbom.sh -O pg_generate_sbom.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./pg_generate_sbom.sh --pg_version=${PG_VERSION} --repo_type=${REPO_TYPE} ${STAGE_PARAM}" + """ +} + +void uploadPGSBOMToTestingDownloadServer(String productName, String packageVersion, String SBOMType) { + + script { + try { + uploadPGSBOMToDownloadsTesting(params.CLOUD, productName, packageVersion, SBOMType) + } catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def TIMESTAMP + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_sbom repository', + name: 'GIT_REPO') + string( + defaultValue: '17.4', + description: 'Version of PostgreSQL server', + name: 'PG_VERSION') + string( + defaultValue: 'main', + description: 'Tag/Branch for postgres packaging repository', + name: 'GIT_BRANCH') + choice( + choices: 'laboratory\ntesting\nexperimental\nrelease', + description: 'Packaging repository type', + name: 'REPO_TYPE') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + + stage('Create timestamp') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + script { + TIMESTAMP = sh(script: 'date +%Y%m%d%H%M%S', returnStdout: true).trim() + sh """ + echo ${TIMESTAMP} > timestamp + cat timestamp + """ + TIMESTAMP = sh(returnStdout: true, script: "cat timestamp").trim() + } + stash includes: 'timestamp', name: 'timestamp' + } + } + + stage('Generate SBOM') { + parallel { + stage('Generate PG SBOM OL/8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("oraclelinux:8", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM OL/8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("oraclelinux:8", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM OL/9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("oraclelinux:9", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM OL/9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("oraclelinux:9", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Focal AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("ubuntu:focal", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Focal ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("ubuntu:focal", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Jammy AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("ubuntu:jammy", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Jammy ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("ubuntu:jammy", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Noble AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("ubuntu:noble", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Noble ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("ubuntu:noble", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Bullseye AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("debian:bullseye", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Bullseye ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("debian:bullseye", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Bookworm AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("debian:bookworm", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Bookworm ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("debian:bookworm", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadPGSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + } //parallel + } // stage + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_generate_sbom.yml b/ppg/pg_generate_sbom.yml new file mode 100644 index 0000000000..32966dd55c --- /dev/null +++ b/ppg/pg_generate_sbom.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pg_generate_sbom + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_generate_sbom.groovy + diff --git a/ppg/pg_percona_telemetry_autobuild.groovy b/ppg/pg_percona_telemetry_autobuild.groovy index 7791e6aba8..cf198f34d9 100644 --- a/ppg/pg_percona_telemetry_autobuild.groovy +++ b/ppg/pg_percona_telemetry_autobuild.groovy @@ -1,5 +1,5 @@ /* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) @@ -24,17 +24,18 @@ void installCli(String PLATFORM) { void buildStage(String DOCKER_OS, String STAGE_PARAM) { sh """ - echo "Docker: $DOCKER_OS, Release: PG$PG_RELEASE, Stage: $STAGE_PARAM" set -o xtrace - mkdir -p test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/main/percona-packaging/scripts/pg_percona_telemetry_builder.sh -O ppt_builder.sh || curl \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/main/percona-packaging/scripts/pg_percona_telemetry_builder.sh -o ppt_builder.sh + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/percona-packaging/scripts/pg_percona_telemetry_builder.sh -O ppt_builder.sh pwd -P + ls -laR export build_dir=\$(pwd -P) - set -o xtrace - cd \${build_dir} - sudo bash -x ./ppt_builder.sh --builddir=\${build_dir}/test --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} --install_deps=1 - bash -x ./ppt_builder.sh --builddir=\${build_dir}/test --version=\${VERSION} --branch=\${BRANCH} --repo=\${GIT_REPO} --rpm_release=\${RPM_RELEASE} --deb_release=\${DEB_RELEASE} --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} "$STAGE_PARAM" - """ + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppt_builder.sh --builddir=\${build_dir}/test --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} --install_deps=1 + bash -x ./ppt_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${VERSION} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --pg_release=${PG_RELEASE} --ppg_repo_name=${PPG_REPO} ${STAGE_PARAM}" + """ } void cleanUpWS() { @@ -47,9 +48,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: '1.0.0', description: 'General version of the product', @@ -78,7 +83,7 @@ pipeline { choice( name: 'PG_RELEASE', description: 'PPG major version to test', - choices: ['11', '12', '13', '14', '15', '16'] + choices: ['11', '12', '13', '14', '15', '16', '17'] ) string( defaultValue: 'ppg-16.0', @@ -98,13 +103,12 @@ pipeline { stages { stage('Download source from github') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo '====> Source will be downloaded from github' //slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") cleanUpWS() - installCli("deb") buildStage("ubuntu:focal", "--get_sources=1") sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg-percona-telemetry.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") @@ -120,170 +124,256 @@ pipeline { } stash includes: 'uploadPath', name: 'uploadPath' stash includes: 'test/pg-percona-telemetry.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } //stage stage('Build percona_pg_telemetry generic source packages') { parallel { stage('Source rpm') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo "====> Build percona_pg_telemetry generic source rpm" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Source deb') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo "====> Build percona_pg_telemetry generic source deb" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } //stage } //parallel } //stage stage('Build percona_pg_telemetry RPMs') { parallel { - stage('OL 8') { + stage('OL 8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + echo "====> Build percona_pg_telemetry rpm on OL 8 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } //stage + stage('OL 8 ARM') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { echo "====> Build percona_pg_telemetry rpm on OL 8 PG${PG_RELEASE}" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } //stage - stage('OL 9') { + stage('OL 9 AMD') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo "====> Build percona_pg_telemetry rpm on OL 9 PG${PG_RELEASE}" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } //stage + stage('OL 9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry rpm on OL 9 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } //stage } //parallel } //stage stage('Build percona_pg_telemetry DEBs') { parallel { - stage('Ubuntu 20.04') { + stage('Ubuntu 20.04 AMD') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo "====> Build percona_pg_telemetry deb on Ubuntu 20.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } //stage - stage('Ubuntu 22.04') { + stage('Ubuntu 20.04 ARM') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Ubuntu 20.04 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Ubuntu 22.04 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo "====> Build percona_pg_telemetry deb on Ubuntu 22.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } //stage - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu 22.04 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Ubuntu 22.04 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo "====> Build percona_pg_telemetry deb on Ubuntu 24.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Ubuntu 24.04 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Debian 11 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + echo "====> Build percona_pg_telemetry deb on Debian 11 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } //stage - stage('Debian 11') { + stage('Debian 11 ARM') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { echo "====> Build percona_pg_telemetry deb on Debian 11 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } //stage - stage('Debian 12') { + stage('Debian 12 AMD') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo "====> Build percona_pg_telemetry deb on Debian 12 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Debian 12 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Debian 12 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } //stage } //parallel } //stage + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } } //stages diff --git a/ppg/pg_percona_telemetry_autobuild.yml b/ppg/pg_percona_telemetry_autobuild.yml index d731397664..f16cfc2893 100644 --- a/ppg/pg_percona_telemetry_autobuild.yml +++ b/ppg/pg_percona_telemetry_autobuild.yml @@ -1,5 +1,5 @@ - job: - name: pg_percona_telemetry-autobuild-RELEASE + name: hetzner-pg_percona_telemetry-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_percona_telemetry_autobuild.groovy diff --git a/ppg/pg_percona_telemetry_autobuild_arm.groovy b/ppg/pg_percona_telemetry_autobuild_arm.groovy new file mode 100644 index 0000000000..c9a38543d6 --- /dev/null +++ b/ppg/pg_percona_telemetry_autobuild_arm.groovy @@ -0,0 +1,300 @@ +/* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +void installCli(String PLATFORM) { + sh """ + set -o xtrace + if [ -d aws ]; then + rm -rf aws + fi + if [ ${PLATFORM} = "deb" ]; then + sudo apt-get update + sudo apt-get -y install wget curl unzip gnupg2 + elif [ ${PLATFORM} = "rpm" ]; then + sudo yum -y install wget curl unzip gnupg2 + fi + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip + unzip awscliv2.zip + sudo ./aws/install || true + """ +} +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/percona-packaging/scripts/pg_percona_telemetry_builder.sh -O ppt_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppt_builder.sh --builddir=\${build_dir}/test --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} --install_deps=1 + bash -x ./ppt_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${VERSION} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --pg_release=${PG_RELEASE} --ppg_repo_name=${PPG_REPO} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: '1.0.0', + description: 'General version of the product', + name: 'VERSION' + ) + string( + defaultValue: 'https://github.com/Percona-Lab/percona_pg_telemetry.git', + description: 'percona_pg_telemetry repo', + name: 'GIT_REPO' + ) + string( + defaultValue: 'main', + description: 'Branch for tests', + name: 'BRANCH' + ) + string( + defaultValue: '1', + description: 'rpm release number', + name: 'RPM_RELEASE' + ) + string( + defaultValue: '1', + description: 'deb release number', + name: 'DEB_RELEASE' + ) + choice( + name: 'PG_RELEASE', + description: 'PPG major version to test', + choices: ['11', '12', '13', '14', '15', '16', '17'] + ) + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental\nrelease', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Download source from github') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + echo '====> Source will be downloaded from github' + //slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("ubuntu:focal", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg-percona-telemetry.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg-percona-telemetry.properties + cat uploadPath + cat awsUploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + stash includes: 'test/pg-percona-telemetry.properties', name: 'properties' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } //stage + stage('Build percona_pg_telemetry generic source packages') { + parallel { + stage('Source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry generic source rpm" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry generic source deb" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_source_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } //stage + } //parallel + } //stage + stage('Build percona_pg_telemetry RPMs') { + parallel { + stage('OL 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry rpm on OL 8 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } //stage + stage('OL 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry rpm on OL 9 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } //stage + } //parallel + } //stage + stage('Build percona_pg_telemetry DEBs') { + parallel { + stage('Ubuntu 20.04') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Ubuntu 20.04 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Ubuntu 22.04') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Ubuntu 22.04 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Ubuntu 24.04 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Debian 11') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Debian 11 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Debian 12') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Debian 12 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + } //parallel + } //stage + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + } //stages + post { + success { + //slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + deleteDir() + echo "Success" + } + failure { + //slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + deleteDir() + echo "Failure" + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + echo "Always" + } + } +} diff --git a/ppg/pg_percona_telemetry_autobuild_arm.yml b/ppg/pg_percona_telemetry_autobuild_arm.yml new file mode 100644 index 0000000000..ae2d3479f8 --- /dev/null +++ b/ppg/pg_percona_telemetry_autobuild_arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pg_percona_telemetry-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_percona_telemetry_autobuild_arm.groovy diff --git a/ppg/pg_source_tarballs.groovy b/ppg/pg_source_tarballs.groovy index bab623f6da..a79a34f72f 100644 --- a/ppg/pg_source_tarballs.groovy +++ b/ppg/pg_source_tarballs.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -20,7 +20,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { String getPostgreSQLVersion(String BRANCH_NAME, String configureFileName) { def packageVersion = sh(script: """ # Download the configure file - if [[ "${BRANCH_NAME}" == *TDE* ]]; then + if echo "${BRANCH_NAME}" | grep -q "release-"; then wget https://raw.githubusercontent.com/Percona-Lab/postgres/${BRANCH_NAME}/configure -O ${configureFileName} else wget https://raw.githubusercontent.com/postgres/postgres/${BRANCH_NAME}/configure -O ${configureFileName} @@ -41,7 +41,7 @@ void uploadTarballToTestingDownloadServer(String tarballDirectory, String packag script { try { - uploadPGTarballToDownloadsTesting(tarballDirectory, packageVersion) + uploadPGTarballToDownloadsTesting(params.CLOUD, tarballDirectory, packageVersion) } catch (err) { echo "Caught: ${err}" currentBuild.result = 'UNSTABLE' @@ -59,10 +59,14 @@ def TIMESTAMP pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_tarballs repository', @@ -89,7 +93,7 @@ pipeline { stage('Create timestamp') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -110,13 +114,13 @@ pipeline { parallel { stage('Build source tarball for PG 17') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() script { def PG_VERSION=17 - def BRANCH_NAME = "TDE_REL_17_STABLE" + def BRANCH_NAME = "release-17.4.1" def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" def PRODUCT="Percona-PostgreSQL-Source-Tarballs" @@ -128,15 +132,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build source tarball for PG 16') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -154,15 +158,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build source tarball for PG 15') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -180,15 +184,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build source tarball for PG 14') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -206,15 +210,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build source tarball for PG 13') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -232,8 +236,8 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } diff --git a/ppg/pg_source_tarballs.yml b/ppg/pg_source_tarballs.yml index 62c71c2e9f..738a316524 100644 --- a/ppg/pg_source_tarballs.yml +++ b/ppg/pg_source_tarballs.yml @@ -1,5 +1,5 @@ - job: - name: pg_source_tarballs-RELEASE + name: hetzner-pg_source_tarballs-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_source_tarballs.groovy diff --git a/ppg/pg_stat_monitor-autobuild-arm.groovy b/ppg/pg_stat_monitor-autobuild-arm.groovy new file mode 100644 index 0000000000..4dbbe72fde --- /dev/null +++ b/ppg/pg_stat_monitor-autobuild-arm.groovy @@ -0,0 +1,267 @@ +/* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/main/percona-packaging/scripts/pg_stat_monitor_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --version=\${VERSION} --branch=\${BRANCH} --repo=\${GIT_REPO} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: '2.0.0', + description: 'General version of the product', + name: 'VERSION' + ) + string( + defaultValue: 'https://github.com/percona/pg_stat_monitor.git', + description: 'pg_stat_monitor repo', + name: 'GIT_REPO' + ) + string( + defaultValue: 'main', + description: 'Branch for tests', + name: 'BRANCH' + ) + string( + defaultValue: '1', + description: 'rpm release number', + name: 'RPM_RELEASE' + ) + string( + defaultValue: '1', + description: 'deb release number', + name: 'DEB_RELEASE' + ) + choice( + name: 'PG_RELEASE', + description: 'PPG major version to test', + choices: ['11', '12', '13', '14', '15', '16', '17'] + ) + string( + defaultValue: 'ppg-16.1', + description: 'PPG repo name', + name: 'PPG_REPO' + ) + choice( + choices: 'laboratory\ntesting\nexperimental\nrelease', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Download source from github') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg-stat-monitor.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg-stat-monitor.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } //stage + stage('Build pg_stat_monitor generic source packages') { + parallel { + stage('Source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_source_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } //stage + stage('Build pg_stat_monitor RPMs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } //parallel + } //stage + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + } //stages + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + deleteDir() + echo "Success" + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + deleteDir() + echo "Failure" + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + echo "Always" + } + } +} diff --git a/ppg/pg_stat_monitor-autobuild-arm.yml b/ppg/pg_stat_monitor-autobuild-arm.yml new file mode 100644 index 0000000000..a0ef8e1bd5 --- /dev/null +++ b/ppg/pg_stat_monitor-autobuild-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pg_stat_monitor-autobuild-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_stat_monitor-autobuild-arm.groovy diff --git a/ppg/pg_stat_monitor-autobuild.groovy b/ppg/pg_stat_monitor-autobuild.groovy index c64f09bce7..cc19f84f01 100644 --- a/ppg/pg_stat_monitor-autobuild.groovy +++ b/ppg/pg_stat_monitor-autobuild.groovy @@ -1,40 +1,22 @@ /* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) - -void installCli(String PLATFORM) { - sh """ - set -o xtrace - if [ -d aws ]; then - rm -rf aws - fi - if [ ${PLATFORM} = "deb" ]; then - sudo apt-get update - sudo apt-get -y install wget curl unzip gnupg2 - elif [ ${PLATFORM} = "rpm" ]; then - sudo yum -y install wget curl unzip gnupg2 - fi - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip - unzip awscliv2.zip - sudo ./aws/install || true - """ -} - void buildStage(String DOCKER_OS, String STAGE_PARAM) { sh """ - echo "Docker: $DOCKER_OS, Release: PG$PG_RELEASE, Stage: $STAGE_PARAM" set -o xtrace - mkdir -p test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/main/percona-packaging/scripts/pg_stat_monitor_builder.sh -O psm_builder.sh || curl \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/main/percona-packaging/scripts/pg_stat_monitor_builder.sh -o psm_builder.sh + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/main/percona-packaging/scripts/pg_stat_monitor_builder.sh -O builder.sh pwd -P + ls -laR export build_dir=\$(pwd -P) - set -o xtrace - cd \${build_dir} - sudo bash -x ./psm_builder.sh --builddir=\${build_dir}/test --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} --install_deps=1 - bash -x ./psm_builder.sh --builddir=\${build_dir}/test --version=\${VERSION} --branch=\${BRANCH} --repo=\${GIT_REPO} --rpm_release=\${RPM_RELEASE} --deb_release=\${DEB_RELEASE} --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} "$STAGE_PARAM" - """ + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --version=\${VERSION} --branch=\${BRANCH} --repo=\${GIT_REPO} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} ${STAGE_PARAM}" + """ } void cleanUpWS() { @@ -47,11 +29,15 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( - defaultValue: '1.0.0', + defaultValue: '2.0.0', description: 'General version of the product', name: 'VERSION' ) @@ -81,9 +67,10 @@ pipeline { choices: ['11', '12', '13', '14', '15', '16', '17'] ) string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-16.1', description: 'PPG repo name', - name: 'PPG_REPO') + name: 'PPG_REPO' + ) choice( choices: 'laboratory\ntesting\nexperimental\nrelease', description: 'Repo component to push packages to', @@ -98,204 +85,260 @@ pipeline { stages { stage('Download source from github') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo '====> Source will be downloaded from github' - slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") cleanUpWS() - installCli("deb") - buildStage("ubuntu:focal", "--get_sources=1") - sh ''' + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg-stat-monitor.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") echo ${REPO_UPLOAD_PATH} > uploadPath echo ${AWS_STASH_PATH} > awsUploadPath cat test/pg-stat-monitor.properties cat uploadPath - cat awsUploadPath - ''' + ''' script { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - stash includes: 'test/pg-stat-monitor.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } //stage stage('Build pg_stat_monitor generic source packages') { parallel { stage('Source rpm') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pg_stat_monitor generic source rpm" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } - stage('Source deb') { + stage('Source deb') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pg_stat_monitor generic source deb" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } - } //stage + } } //parallel } //stage stage('Build pg_stat_monitor RPMs') { parallel { - stage('OL 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pg_stat_monitor rpm on OL 8 PG${PG_RELEASE}" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } - } //stage - stage('OL 9') { + } + stage('Oracle Linux 8 ARM') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pg_stat_monitor rpm on OL 9 PG${PG_RELEASE}" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } - } //stage - } //parallel - } //stage - stage('Build pg_stat_monitor DEBs') { - parallel { - stage('Ubuntu 20.04') { + } + stage('Oracle Linux 9 ARM') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pg_stat_monitor deb on Ubuntu 20.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Ubuntu 22.04') { + } + stage('Ubuntu Focal(20.04) ARM') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pg_stat_monitor deb on Ubuntu 22.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Ubuntu 24.04') { + } + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'min-noble-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pg_stat_monitor deb on Ubuntu 24.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Debian 11') { + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pg_stat_monitor deb on Debian 11 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Debian 12') { + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pg_stat_monitor deb on Debian 12 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage + } + stage('Debian bookworm(12) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } } //parallel } //stage stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } } //stages post { success { - slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") deleteDir() echo "Success" } diff --git a/ppg/pg_stat_monitor-autobuild.yml b/ppg/pg_stat_monitor-autobuild.yml index 9539125fd9..94b70bbcdc 100644 --- a/ppg/pg_stat_monitor-autobuild.yml +++ b/ppg/pg_stat_monitor-autobuild.yml @@ -1,5 +1,5 @@ - job: - name: pg_stat_monitor-autobuild-RELEASE + name: hetzner-pg_stat_monitor-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_stat_monitor-autobuild.groovy diff --git a/ppg/pg_tarballs-arm.groovy b/ppg/pg_tarballs-arm.groovy new file mode 100644 index 0000000000..0b1eeb803d --- /dev/null +++ b/ppg/pg_tarballs-arm.groovy @@ -0,0 +1,402 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_tarballs/pg_tarballs_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --build_dependencies=${BUILD_DEPENDENCIES} ${STAGE_PARAM}" + """ +} + +String getPostgreSQLVersion(String BRANCH_NAME, String configureFileName) { + def packageVersion = sh(script: """ + # Download the configure file + if echo "${BRANCH_NAME}" | grep -q "release-"; then + wget https://raw.githubusercontent.com/Percona-Lab/postgres/${BRANCH_NAME}/configure -O ${configureFileName} + else + wget https://raw.githubusercontent.com/postgres/postgres/${BRANCH_NAME}/configure -O ${configureFileName} + fi + # Read the PACKAGE_VERSION value from the configure file + packageVersion=\$(grep -r 'PACKAGE_VERSION=' ${configureFileName} | tr -dc '[. [:digit:]]') + + # Delete configure file + rm -f ${configureFileName} + + echo "\$packageVersion" + """, returnStdout: true).trim() + + return packageVersion +} + +void uploadTarballToTestingDownloadServer(String tarballDirectory, String packageVersion) { + + script { + try { + uploadPGTarballToDownloadsTesting(params.CLOUD, tarballDirectory, packageVersion) + } catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def TIMESTAMP + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_tarballs repository', + name: 'GIT_REPO') +/* string( + defaultValue: '16.1', + description: 'Version of PostgreSQL server', + name: 'PG_VERSION')*/ + string( + defaultValue: 'main', + description: 'Tag/Branch for pg_tarballs packaging repository', + name: 'GIT_BRANCH') + choice( + choices: '1\n0', + description: 'Build third party dependencies', + name: 'BUILD_DEPENDENCIES') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo destination to push packages to', + name: 'DESTINATION') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + + stage('Create timestamp') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + script { + TIMESTAMP = sh(script: 'date +%Y%m%d%H%M%S', returnStdout: true).trim() + sh """ + echo ${TIMESTAMP} > timestamp + cat timestamp + """ + TIMESTAMP = sh(returnStdout: true, script: "cat timestamp").trim() + } + stash includes: 'timestamp', name: 'timestamp' + } + } + + stage('Build pg_tarballs') { + parallel { + stage('Build pg_tarball 17 for OpenSSL 3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "release-17.5.2" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 17 for OpenSSL 1.1') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = 'release-17.5.2' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 16 for OpenSSL 3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=16 + def BRANCH_NAME = "REL_16_STABLE" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_16="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_16} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 16 for OpenSSL 1.1') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=16 + def BRANCH_NAME = 'REL_16_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_16="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_16} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 15 for OpenSSL 3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=15 + def BRANCH_NAME = 'REL_15_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_15="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_15} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 15 for OpenSSL 1.1') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=15 + def BRANCH_NAME = 'REL_15_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_15="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_15} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 14 for OpenSSL 3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=14 + def BRANCH_NAME = 'REL_14_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_14="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_14} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 14 for OpenSSL 1.1') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=14 + def BRANCH_NAME = 'REL_14_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_14="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_14} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 13 for OpenSSL 3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=13 + def BRANCH_NAME = 'REL_13_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_13="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_13} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 13 for OpenSSL 1.1') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=13 + def BRANCH_NAME = 'REL_13_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_13="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_13} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + } //parallel + } // stage + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_tarballs-arm.yml b/ppg/pg_tarballs-arm.yml new file mode 100644 index 0000000000..e780ee3959 --- /dev/null +++ b/ppg/pg_tarballs-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pg_tarballs-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_tarballs-arm.groovy + diff --git a/ppg/pg_tarballs.groovy b/ppg/pg_tarballs.groovy index f4956712d4..73b1c75f4b 100644 --- a/ppg/pg_tarballs.groovy +++ b/ppg/pg_tarballs.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -20,7 +20,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { String getPostgreSQLVersion(String BRANCH_NAME, String configureFileName) { def packageVersion = sh(script: """ # Download the configure file - if [[ "${BRANCH_NAME}" == *TDE* ]]; then + if echo "${BRANCH_NAME}" | grep -q "release-"; then wget https://raw.githubusercontent.com/Percona-Lab/postgres/${BRANCH_NAME}/configure -O ${configureFileName} else wget https://raw.githubusercontent.com/postgres/postgres/${BRANCH_NAME}/configure -O ${configureFileName} @@ -41,7 +41,7 @@ void uploadTarballToTestingDownloadServer(String tarballDirectory, String packag script { try { - uploadPGTarballToDownloadsTesting(tarballDirectory, packageVersion) + uploadPGTarballToDownloadsTesting(params.CLOUD, tarballDirectory, packageVersion) } catch (err) { echo "Caught: ${err}" currentBuild.result = 'UNSTABLE' @@ -59,10 +59,14 @@ def TIMESTAMP pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_tarballs repository', @@ -93,7 +97,7 @@ pipeline { stage('Create timestamp') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -114,13 +118,13 @@ pipeline { parallel { stage('Build pg_tarball 17 for OpenSSL 3') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() script { def PG_VERSION=17 - def BRANCH_NAME = "TDE_REL_17_STABLE" + def BRANCH_NAME = "release-17.5.2" def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" def PRODUCT="Percona-PostgreSQL-Tarballs" @@ -132,21 +136,21 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 17 for OpenSSL 1.1') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() script { def PG_VERSION=17 - def BRANCH_NAME = 'TDE_REL_17_STABLE' + def BRANCH_NAME = 'release-17.5.2' def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" def PRODUCT="Percona-PostgreSQL-Tarballs" @@ -158,15 +162,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 16 for OpenSSL 3') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -184,15 +188,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 16 for OpenSSL 1.1') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -210,15 +214,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 15 for OpenSSL 3') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -236,15 +240,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 15 for OpenSSL 1.1') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -262,15 +266,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 14 for OpenSSL 3') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -288,15 +292,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 14 for OpenSSL 1.1') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -314,15 +318,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 13 for OpenSSL 3') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -340,15 +344,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 13 for OpenSSL 1.1') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -366,8 +370,8 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } diff --git a/ppg/pg_tarballs.yml b/ppg/pg_tarballs.yml index fe37a9b4da..d51295963b 100644 --- a/ppg/pg_tarballs.yml +++ b/ppg/pg_tarballs.yml @@ -1,5 +1,5 @@ - job: - name: pg_tarballs-RELEASE + name: hetzner-pg_tarballs-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_tarballs.groovy diff --git a/ppg/pg_tarballs_17_nightly.groovy b/ppg/pg_tarballs_17_nightly.groovy new file mode 100644 index 0000000000..41b6a293ae --- /dev/null +++ b/ppg/pg_tarballs_17_nightly.groovy @@ -0,0 +1,246 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_tarballs/pg_tarballs_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --build_dependencies=${BUILD_DEPENDENCIES} ${STAGE_PARAM}" + """ +} + +String getPostgreSQLVersion(String BRANCH_NAME, String configureFileName) { + def packageVersion = sh(script: """ + # Download the configure file + if echo "${BRANCH_NAME}" | grep -q "release-"; then + wget https://raw.githubusercontent.com/Percona-Lab/postgres/${BRANCH_NAME}/configure -O ${configureFileName} + else + wget https://raw.githubusercontent.com/postgres/postgres/${BRANCH_NAME}/configure -O ${configureFileName} + fi + # Read the PACKAGE_VERSION value from the configure file + packageVersion=\$(grep -r 'PACKAGE_VERSION=' ${configureFileName} | tr -dc '[. [:digit:]]') + + # Delete configure file + rm -f ${configureFileName} + + echo "\$packageVersion" + """, returnStdout: true).trim() + + return packageVersion +} + +void uploadTarballToTestingDownloadServer(String tarballDirectory, String packageVersion) { + + script { + try { + uploadPGTarballToDownloadsTesting(params.CLOUD, tarballDirectory, packageVersion) + } catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def TIMESTAMP + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_tarballs repository', + name: 'GIT_REPO') +/* string( + defaultValue: '16.1', + description: 'Version of PostgreSQL server', + name: 'PG_VERSION')*/ + string( + defaultValue: 'main', + description: 'Tag/Branch for pg_tarballs packaging repository', + name: 'GIT_BRANCH') + choice( + choices: '1\n0', + description: 'Build third party dependencies', + name: 'BUILD_DEPENDENCIES') + choice( + choices: 'testing\nlaboratory\nexperimental', + description: 'Repo destination to push packages to', + name: 'DESTINATION') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + + stage('Create timestamp') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + script { + TIMESTAMP = sh(script: 'date +%Y%m%d%H%M%S', returnStdout: true).trim() + sh """ + echo ${TIMESTAMP} > timestamp + cat timestamp + """ + TIMESTAMP = sh(returnStdout: true, script: "cat timestamp").trim() + } + stash includes: 'timestamp', name: 'timestamp' + } + } + + stage('Build pg_tarballs') { + parallel { + stage('Build pg_tarball 17 for OpenSSL 3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "release-17.5.2" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 17 for OpenSSL 1.1') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = 'release-17.5.2' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 17 for OpenSSL 3 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "release-17.5.2" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 17 for OpenSSL 1.1 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = 'release-17.5.2' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + } //parallel + } // stage + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_tarballs_17_nightly.yml b/ppg/pg_tarballs_17_nightly.yml new file mode 100644 index 0000000000..5c4d328cf5 --- /dev/null +++ b/ppg/pg_tarballs_17_nightly.yml @@ -0,0 +1,17 @@ +- job: + name: hetzner-pg_tarballs-nightly + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_tarballs_17_nightly.groovy + triggers: + - timed: "0 0 * * *" + diff --git a/ppg/pg_tde-arm.groovy b/ppg/pg_tde-arm.groovy new file mode 100644 index 0000000000..c658721819 --- /dev/null +++ b/ppg/pg_tde-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_tde/pg_tde_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_tde repository', + name: 'GIT_REPO') + string( + defaultValue: 'main', + description: 'Tag/Branch for pg_tde repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.2', + description: 'Tag/Branch for pg_tde packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.2', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create pg_tde source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg_tde.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg_tde.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build pg_tde generic source packages') { + parallel { + stage('Build pg_tde generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build pg_tde generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build pg_tde RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_tde-arm.yml b/ppg/pg_tde-arm.yml new file mode 100644 index 0000000000..0a2e655e1c --- /dev/null +++ b/ppg/pg_tde-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pg_tde-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_tde-arm.groovy + diff --git a/ppg/pg_tde.groovy b/ppg/pg_tde.groovy index 1d6767c32d..01574edb76 100644 --- a/ppg/pg_tde.groovy +++ b/ppg/pg_tde.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_tde repository', @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build pg_tde generic source packages') { parallel { stage('Build pg_tde generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build pg_tde generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,93 +126,93 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +220,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pg_tde.yml b/ppg/pg_tde.yml index b6f72a92c9..efb9debf24 100644 --- a/ppg/pg_tde.yml +++ b/ppg/pg_tde.yml @@ -1,5 +1,5 @@ - job: - name: pg_tde-RELEASE + name: hetzner-pg_tde-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_tde.groovy diff --git a/ppg/pg_tde_arm_nightly.groovy b/ppg/pg_tde_arm_nightly.groovy new file mode 100644 index 0000000000..2e6f43c89c --- /dev/null +++ b/ppg/pg_tde_arm_nightly.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_tde/pg_tde_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --nightly=1 ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_tde repository', + name: 'GIT_REPO') + string( + defaultValue: 'main', + description: 'Tag/Branch for pg_tde repository', + name: 'PG_BRANCH') + string( + defaultValue: '17.5.2', + description: 'Tag/Branch for pg_tde packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '2', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '2', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-17.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'experimental\nlaboratory\ntesting', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create pg_tde source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg_tde.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg_tde.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build pg_tde generic source packages') { + parallel { + stage('Build pg_tde generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build pg_tde generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build pg_tde RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_tde_arm_nightly.yml b/ppg/pg_tde_arm_nightly.yml new file mode 100644 index 0000000000..5ccb8eae5c --- /dev/null +++ b/ppg/pg_tde_arm_nightly.yml @@ -0,0 +1,16 @@ +- job: + name: hetzner-pg_tde_arm-nightly + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_tde_arm_nightly.groovy + triggers: + - timed: "0 0 * * *" diff --git a/ppg/pg_tde_nightly.groovy b/ppg/pg_tde_nightly.groovy index b4033b3627..b67272d8fd 100644 --- a/ppg/pg_tde_nightly.groovy +++ b/ppg/pg_tde_nightly.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_tde repository', @@ -41,19 +45,19 @@ pipeline { description: 'Tag/Branch for pg_tde repository', name: 'PG_BRANCH') string( - defaultValue: '17.0', + defaultValue: '17.5.2', description: 'Tag/Branch for pg_tde packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', + defaultValue: '2', description: 'RPM release value', name: 'RPM_RELEASE') string( - defaultValue: '1', + defaultValue: '2', description: 'DEB release value', name: 'DEB_RELEASE') string( - defaultValue: 'ppg-17.0', + defaultValue: 'ppg-17.5', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build pg_tde generic source packages') { parallel { stage('Build pg_tde generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build pg_tde generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,93 +126,93 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +220,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pg_tde_nightly.yml b/ppg/pg_tde_nightly.yml index ec8649cef9..09808bfeec 100644 --- a/ppg/pg_tde_nightly.yml +++ b/ppg/pg_tde_nightly.yml @@ -1,5 +1,5 @@ - job: - name: pg_tde-nightly + name: hetzner-pg_tde-nightly project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_tde_nightly.groovy diff --git a/ppg/pgaudit-arm.groovy b/ppg/pgaudit-arm.groovy new file mode 100644 index 0000000000..aa46eb738c --- /dev/null +++ b/ppg/pgaudit-arm.groovy @@ -0,0 +1,253 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgaudit/pgaudit_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_audit repository', + name: 'GIT_REPO') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pg_audit repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.1', + description: 'Tag/Branch for pg_audit packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.1', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PG_AUDIT source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgaudit.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgaudit.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PG_AUDIT generic source packages') { + parallel { + stage('Build PG_AUDIT generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PG_AUDIT generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PG_AUDIT RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgaudit-arm.yml b/ppg/pgaudit-arm.yml new file mode 100644 index 0000000000..fd157e864d --- /dev/null +++ b/ppg/pgaudit-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pgaudit-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgaudit-arm.groovy + diff --git a/ppg/pgaudit.groovy b/ppg/pgaudit.groovy index fb8c84fede..e8ed275eb2 100644 --- a/ppg/pgaudit.groovy +++ b/ppg/pgaudit.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_audit repository', @@ -84,131 +88,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PG_AUDIT generic source packages') { parallel { stage('Build PG_AUDIT generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PG_AUDIT generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PG_AUDIT RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +311,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgaudit.yml b/ppg/pgaudit.yml index aef6bd3500..a2592e23f7 100644 --- a/ppg/pgaudit.yml +++ b/ppg/pgaudit.yml @@ -1,5 +1,5 @@ - job: - name: pgaudit-RELEASE + name: hetzner-pgaudit-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgaudit.groovy diff --git a/ppg/pgaudit_set_user-arm.groovy b/ppg/pgaudit_set_user-arm.groovy new file mode 100644 index 0000000000..3880689cfe --- /dev/null +++ b/ppg/pgaudit_set_user-arm.groovy @@ -0,0 +1,253 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgaudit_set_user/pgaudit_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_audit repository', + name: 'GIT_REPO') + string( + defaultValue: '4.0.1', + description: 'Tag/Branch for pg_audit-set-user', + name: 'PG_BRANCH') + string( + defaultValue: '16.1', + description: 'Tag/Branch for pg_audit-set-user packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.1', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PG_AUDIT source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgaudit.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgaudit.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PG_AUDIT generic source packages') { + parallel { + stage('Build PG_AUDIT generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PG_AUDIT generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PG_AUDIT RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgaudit_set_user-arm.yml b/ppg/pgaudit_set_user-arm.yml new file mode 100644 index 0000000000..b11b4adf86 --- /dev/null +++ b/ppg/pgaudit_set_user-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pgaudit_set_user-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgaudit_set_user-arm.groovy \ No newline at end of file diff --git a/ppg/pgaudit_set_user.groovy b/ppg/pgaudit_set_user.groovy index 5eb8855c0e..da112a1113 100644 --- a/ppg/pgaudit_set_user.groovy +++ b/ppg/pgaudit_set_user.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_audit repository', @@ -84,131 +88,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PG_AUDIT generic source packages') { parallel { stage('Build PG_AUDIT generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PG_AUDIT generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PG_AUDIT RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +311,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgaudit_set_user.yml b/ppg/pgaudit_set_user.yml index 78d1efa15f..9c23675724 100644 --- a/ppg/pgaudit_set_user.yml +++ b/ppg/pgaudit_set_user.yml @@ -1,5 +1,5 @@ - job: - name: pgaudit_set_user-RELEASE + name: hetzner-pgaudit_set_user-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgaudit_set_user.groovy \ No newline at end of file diff --git a/ppg/pgbackrest-arm.groovy b/ppg/pgbackrest-arm.groovy new file mode 100644 index 0000000000..3d81f2618e --- /dev/null +++ b/ppg/pgbackrest-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgbackrest/pgbackrest_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_BACKREST repository', + name: 'GIT_REPO') + string( + defaultValue: 'ver_1.4.4', + description: 'Tag/Branch for postgresql', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pg_BACKREST repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PG_BACKREST source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgbackrest.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgbackrest.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PG_BACKREST generic source packages') { + parallel { + stage('Build PG_BACKREST generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PG_BACKREST generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PG_BACKREST RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgbackrest-arm.yml b/ppg/pgbackrest-arm.yml new file mode 100644 index 0000000000..af2bf133e9 --- /dev/null +++ b/ppg/pgbackrest-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pgbackrest-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgbackrest-arm.groovy diff --git a/ppg/pgbackrest.groovy b/ppg/pgbackrest.groovy index f82f4c24ca..fe933954f2 100644 --- a/ppg/pgbackrest.groovy +++ b/ppg/pgbackrest.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_BACKREST repository', @@ -84,131 +88,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PG_BACKREST generic source packages') { parallel { stage('Build PG_BACKREST generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PG_BACKREST generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PG_BACKREST RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +311,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgbackrest.yml b/ppg/pgbackrest.yml index 8178ce4242..5e3208db0d 100644 --- a/ppg/pgbackrest.yml +++ b/ppg/pgbackrest.yml @@ -1,5 +1,5 @@ - job: - name: pgbackrest-RELEASE + name: hetzner-pgbackrest-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgbackrest.groovy \ No newline at end of file diff --git a/ppg/pgbadger-arm.groovy b/ppg/pgbadger-arm.groovy new file mode 100644 index 0000000000..cf50b02357 --- /dev/null +++ b/ppg/pgbadger-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgbadger/pgbadger_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_badger repository', + name: 'GIT_REPO') + string( + defaultValue: 'ver_1.4.4', + description: 'Tag/Branch for postgresql', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pg_badger repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PG_badger source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgbadger.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgbadger.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PG_badger generic source packages') { + parallel { + stage('Build PG_badger generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PG_badger generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PG_badger RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgbadger-arm.yml b/ppg/pgbadger-arm.yml new file mode 100644 index 0000000000..22773e6dd0 --- /dev/null +++ b/ppg/pgbadger-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pgbadger-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgbadger-arm.groovy diff --git a/ppg/pgbadger.groovy b/ppg/pgbadger.groovy index 0556a563c4..b742810d7f 100644 --- a/ppg/pgbadger.groovy +++ b/ppg/pgbadger.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_badger repository', @@ -84,131 +88,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PG_badger generic source packages') { parallel { stage('Build PG_badger generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PG_badger generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PG_badger RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +311,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgbadger.yml b/ppg/pgbadger.yml index 15816a7e91..3a50b2916f 100644 --- a/ppg/pgbadger.yml +++ b/ppg/pgbadger.yml @@ -1,5 +1,5 @@ - job: - name: pgbadger-RELEASE + name: hetzner-pgbadger-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgbadger.groovy \ No newline at end of file diff --git a/ppg/pgbouncer-arm.groovy b/ppg/pgbouncer-arm.groovy new file mode 100644 index 0000000000..ce4f866a64 --- /dev/null +++ b/ppg/pgbouncer-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgbouncer/pgbouncer_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_bouncer repository', + name: 'GIT_REPO') + string( + defaultValue: 'ver_1.4.4', + description: 'Tag/Branch for postgresql', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pg_bouncer repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PG_bouncer source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgbouncer.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgbouncer.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PG_bouncer generic source packages') { + parallel { + stage('Build PG_bouncer generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PG_bouncer generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PG_bouncer RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgbouncer-arm.yml b/ppg/pgbouncer-arm.yml new file mode 100644 index 0000000000..4fa82e885b --- /dev/null +++ b/ppg/pgbouncer-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pgbouncer-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgbouncer-arm.groovy diff --git a/ppg/pgbouncer.groovy b/ppg/pgbouncer.groovy index 544dfda8fb..05ccfce8de 100644 --- a/ppg/pgbouncer.groovy +++ b/ppg/pgbouncer.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_bouncer repository', @@ -84,131 +88,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PG_bouncer generic source packages') { parallel { stage('Build PG_bouncer generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PG_bouncer generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PG_bouncer RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +311,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgbouncer.yml b/ppg/pgbouncer.yml index 1fc522eb90..06d128a77b 100644 --- a/ppg/pgbouncer.yml +++ b/ppg/pgbouncer.yml @@ -1,5 +1,5 @@ - job: - name: pgbouncer-RELEASE + name: hetzner-pgbouncer-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgbouncer.groovy \ No newline at end of file diff --git a/ppg/pgpool-arm.groovy b/ppg/pgpool-arm.groovy new file mode 100644 index 0000000000..3282605f04 --- /dev/null +++ b/ppg/pgpool-arm.groovy @@ -0,0 +1,269 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_BUILD_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BUILD_BRANCH}/pgpool2/pgpool2_builder.sh -O pgpool2_builder.sh + pwd -P + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./pgpool2_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./pgpool2_builder.sh --builddir=\${build_dir}/test --branch=\${BRANCH} --repo=\${GIT_REPO} --pp_branch=\${BUILD_BRANCH} --pp_repo=\${GIT_BUILD_REPO} --rpm_release=\${RPM_RELEASE} --deb_release=\${DEB_RELEASE} --pg_release=\${PG_RELEASE} $STAGE_PARAM" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: '4.4.2', + description: 'General version of the product', + name: 'VERSION' + ) + string( + defaultValue: 'https://git.postgresql.org/git/pgpool2.git', + description: 'pgpool2 repo', + name: 'GIT_REPO' + ) + string( + defaultValue: 'V4_4_STABLE', + description: 'Branch for pgpool2 repo', + name: 'BRANCH' + ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'Build pgpool2 repo', + name: 'GIT_BUILD_REPO' + ) + string( + defaultValue: 'main', + description: 'Branch for build repo', + name: 'BUILD_BRANCH' + ) + string( + defaultValue: '1', + description: 'rpm release number', + name: 'RPM_RELEASE' + ) + string( + defaultValue: '1', + description: 'deb release number', + name: 'DEB_RELEASE' + ) + string( + defaultValue: '16.1', + description: 'PPG major version to test', + name: 'PG_RELEASE' + ) + choice( + choices: 'laboratory\ntesting\nexperimental\nrelease', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create pgpool2 source tarball') { + steps { + slackNotify("", "#00FF00", "[${JOB_NAME}]: starting build for ${BUILD_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgpool2.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgpool2.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build pgpool2 generic source packages') { + parallel { + stage('Build pgpool2 generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build pgpool2 generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_source_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build pgpool2 RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, "ppg-${PG_RELEASE}", COMPONENT) + } + } + + } + post { + success { + slackNotify("", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${BUILD_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${BUILD_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("", "#FF0000", "[${JOB_NAME}]: build failed for ${BUILD_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgpool-arm.yml b/ppg/pgpool-arm.yml new file mode 100644 index 0000000000..33d386d847 --- /dev/null +++ b/ppg/pgpool-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pgpool-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgpool-arm.groovy diff --git a/ppg/pgpool2-build-RELEASE.yml b/ppg/pgpool2-build-RELEASE.yml index ba54061b7e..a562beb9eb 100644 --- a/ppg/pgpool2-build-RELEASE.yml +++ b/ppg/pgpool2-build-RELEASE.yml @@ -1,5 +1,5 @@ - job: - name: pgpool2-build-RELEASE + name: hetzner-pgpool2-build-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgpool2-build.groovy diff --git a/ppg/pgpool2-build.groovy b/ppg/pgpool2-build.groovy index 2ec22a575d..6c4fb5d091 100644 --- a/ppg/pgpool2-build.groovy +++ b/ppg/pgpool2-build.groovy @@ -1,76 +1,21 @@ -/* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' -]) - -void installCli(String PLATFORM) { - sh """ - set -o xtrace - if [ -d aws ]; then - rm -rf aws - fi - if [ ${PLATFORM} = "deb" ]; then - sudo apt-get update - sudo apt-get -y install wget curl unzip gnupg2 - elif [ ${PLATFORM} = "rpm" ]; then - sudo yum -y install wget curl unzip gnupg2 - fi - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip - unzip awscliv2.zip - sudo ./aws/install || true - """ -} +]) _ void buildStage(String DOCKER_OS, String STAGE_PARAM) { sh """ - echo "Docker: $DOCKER_OS, Release: PG$PG_RELEASE, Stage: $STAGE_PARAM" set -o xtrace - mkdir -p test - wget \$(echo ${GIT_BUILD_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BUILD_BRANCH}/pgpool2/pgpool2_builder.sh -O pg2_builder.sh || curl \$(echo ${GIT_BUILD_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BUILD_BRANCH}/pgpool2/pgpool2_builder.sh -o pg2_builder.sh - #git clone ${GIT_BUILD_REPO} - #cd postgres-packaging - #git checkout ${BUILD_BRANCH} - #cd .. - #cp postgres-packaging/pgpool2/pgpool2_builder.sh pg2_builder.sh - if [ -f /etc/redhat-release ]; then - sudo yum install -y https://repo.percona.com/yum/percona-release-latest.noarch.rpm || true - else - DEBIAN_VERSION=\$(lsb_release -sc) - # sed -e '/.*backports.*/d' /etc/apt/sources.list > sources.list.new - # sudo mv -vf sources.list.new /etc/apt/sources.list - if [ \${DEBIAN_VERSION} = bionic ]; then - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main" | sudo tee -a /etc/apt/sources.list - fi - if [ \${DEBIAN_VERSION} = buster ]; then - sudo apt-get -y update --allow-releaseinfo-change || true - fi - until sudo apt-get update; do - sleep 30 - echo "Waiting ..." - done - until sudo apt-get -y install gpgv curl clang gnupg; do - sleep 30 - echo "Waiting ..." - done - sudo wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb - sudo dpkg -i percona-release_latest.generic_all.deb - fi - sudo percona-release enable ppg-${PG_RELEASE} release + mkdir test + wget \$(echo ${GIT_BUILD_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BUILD_BRANCH}/pgpool2/pgpool2_builder.sh -O pgpool2_builder.sh pwd -P export build_dir=\$(pwd -P) - set -o xtrace - cd \${build_dir} - if [ -f ./test/pgpool2.properties ]; then - . ./test/pgpool2.properties - fi - sed -i "s:VERSION=\\"1.0.0:VERSION=\\"$VERSION:" pg2_builder.sh - sed -i "s:PG_RELEASE=11:PG_RELEASE=\"${PG_RELEASE}\":" pg2_builder.sh - - sudo bash -x ./pg2_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./pg2_builder.sh --builddir=\${build_dir}/test --branch=\${BRANCH} --repo=\${GIT_REPO} --pp_branch=\${BUILD_BRANCH} --pp_repo=\${GIT_BUILD_REPO} --rpm_release=\${RPM_RELEASE} --deb_release=\${DEB_RELEASE} --pg_release=\${PG_RELEASE} "$STAGE_PARAM" - """ + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./pgpool2_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./pgpool2_builder.sh --builddir=\${build_dir}/test --branch=\${BRANCH} --repo=\${GIT_REPO} --pp_branch=\${BUILD_BRANCH} --pp_repo=\${GIT_BUILD_REPO} --rpm_release=\${RPM_RELEASE} --deb_release=\${DEB_RELEASE} --pg_release=\${PG_RELEASE} $STAGE_PARAM" + """ } void cleanUpWS() { @@ -83,9 +28,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: '4.4.2', description: 'General version of the product', @@ -130,228 +79,282 @@ pipeline { choices: 'laboratory\ntesting\nexperimental\nrelease', description: 'Repo component to push packages to', name: 'COMPONENT') - } + } options { skipDefaultCheckout() disableConcurrentBuilds() buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) - timestamps () } stages { - stage('Download source from github') { - agent { - label 'min-focal-x64' - } + stage('Create pgpool2 source tarball') { steps { - echo '====> Source will be downloaded from github' - // slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + slackNotify("", "#00FF00", "[${JOB_NAME}]: starting build for ${BUILD_BRANCH} - [${BUILD_URL}]") cleanUpWS() - installCli("deb") - buildStage("ubuntu:focal", "--get_sources=1") - sh ''' + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgpool2.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") echo ${REPO_UPLOAD_PATH} > uploadPath echo ${AWS_STASH_PATH} > awsUploadPath cat test/pgpool2.properties cat uploadPath - cat awsUploadPath - ''' + ''' script { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - stash includes: 'test/pgpool2.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } - } //stage + } stage('Build pgpool2 generic source packages') { parallel { - stage('Source rpm') { + stage('Build pgpool2 generic source rpm') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pgpool2 generic source rpm" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } - stage('Source deb') { + stage('Build pgpool2 generic source deb') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pgpool2 generic source deb" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } - } //stage + } } //parallel - } //stage - stage('Build pgpool2 RPMs') { + } // stage + stage('Build pgpool2 RPMs/DEBs/Binary tarballs') { parallel { - stage('OL 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pgpool2 rpm on Centos 8 PG${PG_RELEASE}" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } - } //stage - stage('OL 9') { + } + stage('Oracle Linux 8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pgpool2 rpm on OL9 PG${PG_RELEASE}" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } - } //stage - } //parallel - } //stage - stage('Build pgpool2 DEBs') { - parallel { - stage('Ubuntu 20.04') { + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pgpool2 deb on Ubuntu 20.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Ubuntu 22.04') { + } + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pgpool2 deb on Ubuntu 22.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Ubuntu 24.04') { + } + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'min-noble-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pgpool2 deb on Ubuntu 24.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Debian 11') { + } + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pgpool2 deb on Debian 11 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Debian 12') { + } + stage('Debian Bullseye(11) ARM') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pgpool2 deb on Debian 11 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - } //parallel - } //stage + } + } + } + stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild("ppg-${PG_RELEASE}", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "ppg-${PG_RELEASE}", COMPONENT) } } - } //stages + + } post { success { -// slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") - deleteDir() - echo "Success" + slackNotify("", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${BUILD_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${BUILD_BRANCH}" + } + deleteDir() } failure { -// slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") - deleteDir() - echo "Failure" + slackNotify("", "#FF0000", "[${JOB_NAME}]: build failed for ${BUILD_BRANCH} - [${BUILD_URL}]") + deleteDir() } always { sh ''' sudo rm -rf ./* ''' deleteDir() - echo "Always" } } } diff --git a/ppg/pgrepack.groovy b/ppg/pgrepack.groovy index cd24c62427..031f4a3988 100644 --- a/ppg/pgrepack.groovy +++ b/ppg/pgrepack.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_repack repository', @@ -84,131 +88,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PG_REPACK generic source packages') { parallel { stage('Build PG_REPACK generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PG_REPACK generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PG_REPACK RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +311,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgrepack.yml b/ppg/pgrepack.yml index 02c9d26d7e..e7b0c3a7c2 100644 --- a/ppg/pgrepack.yml +++ b/ppg/pgrepack.yml @@ -1,5 +1,5 @@ - job: - name: pgrepack-RELEASE + name: hetzner-pgrepack-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgrepack.groovy diff --git a/ppg/pgrepack_arm.groovy b/ppg/pgrepack_arm.groovy new file mode 100644 index 0000000000..b85e4851b5 --- /dev/null +++ b/ppg/pgrepack_arm.groovy @@ -0,0 +1,253 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_repack/pgrepack_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_repack repository', + name: 'GIT_REPO') + string( + defaultValue: 'ver_1.4.4', + description: 'General version of the product', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pg_repack repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.1', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PG_REPACK source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg_repack.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg_repack.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PG_REPACK generic source packages') { + parallel { + stage('Build PG_REPACK generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PG_REPACK generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PG_REPACK RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgrepack_arm.yml b/ppg/pgrepack_arm.yml new file mode 100644 index 0000000000..cad913b5f3 --- /dev/null +++ b/ppg/pgrepack_arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pgrepack-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgrepack_arm.groovy \ No newline at end of file diff --git a/ppg/pgvector-arm.groovy b/ppg/pgvector-arm.groovy new file mode 100644 index 0000000000..3dd6884fa4 --- /dev/null +++ b/ppg/pgvector-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgvector/pgvector_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pgvector repository', + name: 'GIT_REPO') + string( + defaultValue: 'ver_1.4.4', + description: 'Tag/Branch for pgvector repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pgvector packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create pgvector source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgvector.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgvector.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build pgvector generic source packages') { + parallel { + stage('Build pgvector generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build pgvector generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build pgvector RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgvector-arm.yml b/ppg/pgvector-arm.yml new file mode 100644 index 0000000000..8de028f67f --- /dev/null +++ b/ppg/pgvector-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pgvector-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgvector-arm.groovy + diff --git a/ppg/pgvector.groovy b/ppg/pgvector.groovy index 1e8251ecd2..8eb335a262 100644 --- a/ppg/pgvector.groovy +++ b/ppg/pgvector.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pgvector repository', @@ -84,131 +88,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build pgvector generic source packages') { parallel { stage('Build pgvector generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build pgvector generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build pgvector RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +311,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgvector.yml b/ppg/pgvector.yml index bc31e98e3f..2779caff35 100644 --- a/ppg/pgvector.yml +++ b/ppg/pgvector.yml @@ -1,5 +1,5 @@ - job: - name: pgvector-RELEASE + name: hetzner-pgvector-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgvector.groovy diff --git a/ppg/postgis_tarballs.groovy b/ppg/postgis_tarballs.groovy index fd0bb8ec91..9e689d94cd 100644 --- a/ppg/postgis_tarballs.groovy +++ b/ppg/postgis_tarballs.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -27,7 +27,7 @@ void uploadTarballToTestingDownloadServer(String tarballDirectory, String packag script { try { - uploadPGTarballToDownloadsTesting(tarballDirectory, packageVersion) + uploadPGTarballToDownloadsTesting(params.CLOUD, tarballDirectory, packageVersion) } catch (err) { echo "Caught: ${err}" currentBuild.result = 'UNSTABLE' @@ -40,8 +40,8 @@ void buildTarball(String platform, String architecture){ script { unstash "uploadPath-${PG_VERSION}" buildStage("${PG_VERSION}", platform, architecture) - pushArtifactFolder("postgis_output/", AWS_STASH_PATH) - uploadPGTarballfromAWS("postgis_output/", AWS_STASH_PATH, "binary", "${PG_VERSION}") + pushArtifactFolder(params.CLOUD, "postgis_output/", AWS_STASH_PATH) + uploadPGTarballfromAWS(params.CLOUD, "postgis_output/", AWS_STASH_PATH, "binary", "${PG_VERSION}") uploadTarballToTestingDownloadServer("postgis_tarballs", "${PG_VERSION}") } } @@ -56,10 +56,14 @@ def TIMESTAMP pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) //string( // defaultValue: 'https://github.com/percona/postgis-tarballs.git', // description: 'URL for postgis-tarballs repository', @@ -86,7 +90,7 @@ pipeline { stage('Create timestamp') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${PG_VERSION} - [${BUILD_URL}]") @@ -114,7 +118,7 @@ pipeline { parallel { stage('Build postgis-tarball for OL8 amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -125,7 +129,7 @@ pipeline { } stage('Build postgis-tarball for OL8 arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -136,7 +140,7 @@ pipeline { } stage('Build postgis-tarball for OL9 amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -147,7 +151,7 @@ pipeline { } stage('Build postgis-tarball for OL9 arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -158,7 +162,7 @@ pipeline { } stage('Build postgis-tarball for Ubuntu focal amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -169,7 +173,7 @@ pipeline { } stage('Build postgis-tarball for Ubuntu focal arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -180,7 +184,7 @@ pipeline { } stage('Build postgis-tarball for Ubuntu jammy amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -191,7 +195,7 @@ pipeline { } stage('Build postgis-tarball for Ubuntu jammy arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -202,7 +206,7 @@ pipeline { } stage('Build postgis-tarball for Ubuntu noble amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -213,7 +217,7 @@ pipeline { } stage('Build postgis-tarball for Ubuntu noble arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -224,7 +228,7 @@ pipeline { } stage('Build postgis-tarball for Debian bullseye amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -235,7 +239,7 @@ pipeline { } stage('Build postgis-tarball for Debian bullseye arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -246,7 +250,7 @@ pipeline { } stage('Build postgis-tarball for Debian bookworm amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -257,7 +261,7 @@ pipeline { } stage('Build postgis-tarball for Debian bookworm arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() diff --git a/ppg/postgis_tarballs.yml b/ppg/postgis_tarballs.yml index 970930e542..f6a0f11a60 100644 --- a/ppg/postgis_tarballs.yml +++ b/ppg/postgis_tarballs.yml @@ -1,5 +1,5 @@ - job: - name: postgis_tarballs-RELEASE + name: hetzner-postgis_tarballs-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/postgis_tarballs.groovy diff --git a/ppg/postgresql-common.groovy b/ppg/postgresql-common.groovy index c25ea38265..d8545a15c4 100644 --- a/ppg/postgresql-common.groovy +++ b/ppg/postgresql-common.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for postgresql-common repository', @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build postgresql-common generic source packages') { parallel { stage('Build postgresql-common generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build postgresql-common generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,93 +126,93 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +220,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/postgresql-common.yml b/ppg/postgresql-common.yml index f0f10482dc..f501be8a68 100644 --- a/ppg/postgresql-common.yml +++ b/ppg/postgresql-common.yml @@ -1,5 +1,5 @@ - job: - name: postgresql-common-RELEASE + name: hetzner-postgresql-common-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/postgresql-common.groovy diff --git a/ppg/postgresql-ivee-arm.groovy b/ppg/postgresql-ivee-arm.groovy new file mode 100644 index 0000000000..9dc81ac7bc --- /dev/null +++ b/ppg/postgresql-ivee-arm.groovy @@ -0,0 +1,190 @@ +library changelog: false, identifier: "lib@master", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label 'docker-32gb-aarch64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') + string(name: 'PPG_VERSION', defaultValue: '12.19-1', description: 'PPG version') + choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') + choice(name: 'LATEST', choices: ['no','yes'], description: 'Tag image as latest') + } + options { + disableConcurrentBuilds() + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PPG_REPO}-${params.PPG_VERSION}" + } + } + } + stage ('Build image') { + steps { + sh """ + MAJ_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f1 -d'.') + echo \$MAJ_VER + MIN_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f2 -d'.') + echo \$MIN_VER + git clone https://github.com/Percona-Lab/postgresql-docker-ivee + cd postgresql-docker-ivee + sed -E "s/ENV PPG_REPO (.+)/ENV PPG_REPO ${params.PPG_REPO}/" -i Dockerfile + sed -E "s/ENV PPG_MAJOR_VERSION (.+)/ENV PPG_MAJOR_VERSION \$MAJ_VER/" -i Dockerfile + sed -E "s/ENV PPG_MINOR_VERSION (.+)/ENV PPG_MINOR_VERSION \$MIN_VER/" -i Dockerfile + docker build . -t postgresql-ivee -f Dockerfile + """ + } + } + stage ('Push images to perconalab') { + when { + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + docker login -u '${USER}' -p '${PASS}' + MAJ_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f1 -d'.') + MIN_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f2 -d'.') + docker tag postgresql-ivee perconalab/postgresql-ivee:${params.PPG_VERSION}-arm64 + docker tag postgresql-ivee perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 + docker tag postgresql-ivee perconalab/postgresql-ivee:\$MAJ_VER-arm64 + docker push perconalab/postgresql-ivee:${params.PPG_VERSION}-arm64 + docker push perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 + docker push perconalab/postgresql-ivee:\$MAJ_VER-arm64 + + docker manifest create --amend perconalab/postgresql-ivee:${params.PPG_VERSION} \ + perconalab/postgresql-ivee:${params.PPG_VERSION}-amd64 \ + perconalab/postgresql-ivee:${params.PPG_VERSION}-arm64 + docker manifest annotate perconalab/postgresql-ivee:${params.PPG_VERSION} \ + perconalab/postgresql-ivee:${params.PPG_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/postgresql-ivee:${params.PPG_VERSION} \ + perconalab/postgresql-ivee:${params.PPG_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/postgresql-ivee:${params.PPG_VERSION} + docker manifest push perconalab/postgresql-ivee:${params.PPG_VERSION} + + docker manifest create --amend perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER \ + perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER-amd64 \ + perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 + docker manifest annotate perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER \ + perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER \ + perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER + docker manifest push perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER + + docker manifest create --amend perconalab/postgresql-ivee:\$MAJ_VER \ + perconalab/postgresql-ivee:\$MAJ_VER-amd64 \ + perconalab/postgresql-ivee:\$MAJ_VER-arm64 + docker manifest annotate perconalab/postgresql-ivee:\$MAJ_VER \ + perconalab/postgresql-ivee:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/postgresql-ivee:\$MAJ_VER \ + perconalab/postgresql-ivee:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/postgresql-ivee:\$MAJ_VER + docker manifest push perconalab/postgresql-ivee:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend perconalab/postgresql-ivee:latest \ + perconalab/postgresql-ivee:\$MAJ_VER-amd64 \ + perconalab/postgresql-ivee:\$MAJ_VER-arm64 + docker manifest annotate perconalab/postgresql-ivee:latest \ + perconalab/postgresql-ivee:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/postgresql-ivee:latest \ + perconalab/postgresql-ivee:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/postgresql-ivee:latest + docker manifest push perconalab/postgresql-ivee:latest + fi + + """ + } + } + } + stage ('Push images to percona') { + when { + environment name: 'TARGET_REPO', value: 'DockerHub' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + docker login -u '${USER}' -p '${PASS}' + MAJ_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f1 -d'.') + MIN_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f2 -d'.') + docker tag postgresql-ivee percona/postgresql-ivee:${params.PPG_VERSION}-arm64 + docker tag postgresql-ivee percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 + docker tag postgresql-ivee percona/postgresql-ivee:\$MAJ_VER-arm64 + docker push percona/postgresql-ivee:${params.PPG_VERSION}-arm64 + docker push percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 + docker push percona/postgresql-ivee:\$MAJ_VER-arm64 + + docker manifest create --amend percona/postgresql-ivee:${params.PPG_VERSION} \ + percona/postgresql-ivee:${params.PPG_VERSION}-amd64 \ + percona/postgresql-ivee:${params.PPG_VERSION}-arm64 + docker manifest annotate percona/postgresql-ivee:${params.PPG_VERSION} \ + percona/postgresql-ivee:${params.PPG_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/postgresql-ivee:${params.PPG_VERSION} \ + percona/postgresql-ivee:${params.PPG_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect percona/postgresql-ivee:${params.PPG_VERSION} + docker manifest push percona/postgresql-ivee:${params.PPG_VERSION} + + docker manifest create --amend percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER \ + percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER-amd64 \ + percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 + docker manifest annotate percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER \ + percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER \ + percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER + docker manifest push percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER + + docker manifest create --amend percona/postgresql-ivee:\$MAJ_VER \ + percona/postgresql-ivee:\$MAJ_VER-amd64 \ + percona/postgresql-ivee:\$MAJ_VER-arm64 + docker manifest annotate percona/postgresql-ivee:\$MAJ_VER \ + percona/postgresql-ivee:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/postgresql-ivee:\$MAJ_VER \ + percona/postgresql-ivee:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/postgresql-ivee:\$MAJ_VER + docker manifest push percona/postgresql-ivee:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend percona/postgresql-ivee:latest \ + percona/postgresql-ivee:\$MAJ_VER-amd64 \ + percona/postgresql-ivee:\$MAJ_VER-arm64 + docker manifest annotate percona/postgresql-ivee:latest \ + percona/postgresql-ivee:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/postgresql-ivee:latest \ + percona/postgresql-ivee:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/postgresql-ivee:latest + docker manifest push percona/postgresql-ivee:latest + fi + """ + } + } + } + } + post { + always { + sh """ + sudo docker rmi -f \$(sudo docker images -q | uniq) || true + sudo rm -rf ${WORKSPACE}/* + """ + deleteDir() + } + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: Building of PPG ${PPG_VERSION} for ARM, repo ${PPG_REPO} succeed") + } + unstable { + slackNotify("#releases-ci", "#F6F930", "[${JOB_NAME}]: Building of PPG ${PPG_VERSION} fro ARM, repo ${PPG_REPO} unstable - [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: Building of PPG ${PPG_VERSION} for ARM, repo ${PPG_REPO} failed - [${BUILD_URL}]") + } + } +} diff --git a/ppg/postgresql-ivee-arm.yml b/ppg/postgresql-ivee-arm.yml new file mode 100644 index 0000000000..7e4bc1b878 --- /dev/null +++ b/ppg/postgresql-ivee-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-postgresql-ivee-docker-arm + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/postgresql-ivee-arm.groovy diff --git a/ppg/postgresql_common_arm.groovy b/ppg/postgresql_common_arm.groovy new file mode 100644 index 0000000000..73431c8f5b --- /dev/null +++ b/ppg/postgresql_common_arm.groovy @@ -0,0 +1,256 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/postgres-common/ppg-common_builder.sh -O ppg-server_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for ppg-server repository', + name: 'GIT_REPO') + string( + defaultValue: '256', + description: 'Tag/Branch for postgresql-common', + name: 'PG_BRANCH') + string( + defaultValue: '16.1', + description: 'Tag/Branch for ppg-server repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.1', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PPG_SERVER source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-postgresql.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/percona-postgresql.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PPG-SERVER generic source packages') { + parallel { + stage('Build PPG-SERVER generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build postgresql-common generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PPG-SERVER RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/postgresql_common_arm.yml b/ppg/postgresql_common_arm.yml new file mode 100644 index 0000000000..703baa26e9 --- /dev/null +++ b/ppg/postgresql_common_arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-postgresql-common-arm-autobuild-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/postgresql_common_arm.groovy + diff --git a/ppg/postgresql_server.groovy b/ppg/postgresql_server.groovy index 7430e2ff0a..fa1825c928 100644 --- a/ppg/postgresql_server.groovy +++ b/ppg/postgresql_server.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for ppg-server repository', @@ -84,131 +88,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PPG-SERVER generic source packages') { parallel { stage('Build PPG-SERVER generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PPG-SERVER generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PPG-SERVER RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +311,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/postgresql_server.yml b/ppg/postgresql_server.yml index 56c1629bac..af42fc261c 100644 --- a/ppg/postgresql_server.yml +++ b/ppg/postgresql_server.yml @@ -1,5 +1,5 @@ - job: - name: postgresql-server-autobuild-RELEASE + name: hetzner-postgresql-server-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/postgresql_server.groovy diff --git a/ppg/postgresql_server_17_nightly.groovy b/ppg/postgresql_server_17_nightly.groovy new file mode 100644 index 0000000000..cefea961b4 --- /dev/null +++ b/ppg/postgresql_server_17_nightly.groovy @@ -0,0 +1,345 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/postgres/ppg_builder.sh -O ppg-server_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for ppg-server repository', + name: 'GIT_REPO') + string( + defaultValue: 'release-17.5.2', + description: 'Tag/Branch for postgresql', + name: 'PG_BRANCH') + string( + defaultValue: '17.5.2', + description: 'Tag/Branch for ppg-server repository', + name: 'GIT_BRANCH') + string( + defaultValue: '2', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '2', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-17.5', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'testing\nlaboratory\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PPG_SERVER source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-postgresql.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/percona-postgresql.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PPG-SERVER generic source packages') { + parallel { + stage('Build PPG-SERVER generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PPG-SERVER generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PPG-SERVER RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/postgresql_server_17_nightly.yml b/ppg/postgresql_server_17_nightly.yml new file mode 100644 index 0000000000..93556dd19f --- /dev/null +++ b/ppg/postgresql_server_17_nightly.yml @@ -0,0 +1,17 @@ +- job: + name: hetzner-postgresql-server-17_nightly + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/postgresql_server_17_nightly.groovy + triggers: + - timed: "0 0 * * *" + diff --git a/ppg/postgresql_server_arm.groovy b/ppg/postgresql_server_arm.groovy new file mode 100644 index 0000000000..01e778196e --- /dev/null +++ b/ppg/postgresql_server_arm.groovy @@ -0,0 +1,253 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/postgres/ppg_builder.sh -O ppg-server_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for ppg-server repository', + name: 'GIT_REPO') + string( + defaultValue: 'REL_15_RELEASE', + description: 'Tag/Branch for postgresql', + name: 'PG_BRANCH') + string( + defaultValue: '15.3', + description: 'Tag/Branch for ppg-server repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-15.3', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PPG_SERVER source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-postgresql.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/percona-postgresql.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PPG-SERVER generic source packages') { + parallel { + stage('Build PPG-SERVER generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PPG-SERVER generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PPG-SERVER RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/postgresql_server_arm.yml b/ppg/postgresql_server_arm.yml new file mode 100644 index 0000000000..ff9e0e50d0 --- /dev/null +++ b/ppg/postgresql_server_arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-postgresql-server-arm_autobuild-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/postgresql_server_arm.groovy + diff --git a/ppg/postgresql_server_nightly.groovy b/ppg/postgresql_server_nightly.groovy index 94867da246..a7f53948b2 100644 --- a/ppg/postgresql_server_nightly.groovy +++ b/ppg/postgresql_server_nightly.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for ppg-server repository', @@ -41,19 +45,19 @@ pipeline { description: 'Tag/Branch for postgresql', name: 'PG_BRANCH') string( - defaultValue: '17.0', + defaultValue: '17.5.2', description: 'Tag/Branch for ppg-server repository', name: 'GIT_BRANCH') string( - defaultValue: '1', + defaultValue: '2', description: 'RPM release value', name: 'RPM_RELEASE') string( - defaultValue: '1', + defaultValue: '2', description: 'DEB release value', name: 'DEB_RELEASE') string( - defaultValue: 'ppg-17.0', + defaultValue: 'ppg-17.5', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -106,8 +110,8 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PPG-SERVER generic source packages') { @@ -117,28 +121,28 @@ pipeline { parallel { stage('Build PPG-SERVER generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PPG-SERVER generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -150,28 +154,28 @@ pipeline { parallel { stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -182,8 +186,8 @@ pipeline { expression { currentBuild.result != 'NOT_BUILT' } } steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { @@ -192,7 +196,7 @@ pipeline { } steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/postgresql_server_nightly.yml b/ppg/postgresql_server_nightly.yml index 5182d83774..f9d623b8e0 100644 --- a/ppg/postgresql_server_nightly.yml +++ b/ppg/postgresql_server_nightly.yml @@ -1,5 +1,5 @@ - job: - name: postgresql-server-nightly + name: hetzner-postgresql-server-nightly project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/postgresql_server_nightly.groovy diff --git a/ppg/ppg-docker-arm.groovy b/ppg/ppg-docker-arm.groovy new file mode 100644 index 0000000000..ee371c2646 --- /dev/null +++ b/ppg/ppg-docker-arm.groovy @@ -0,0 +1,192 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') + choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') + string(name: 'PPG_VERSION', defaultValue: '12.19-1', description: 'PPG version') + choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') + choice(name: 'LATEST', choices: ['no','yes'], description: 'Tag image as latest') + } + options { + disableConcurrentBuilds() + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PPG_REPO}-${params.PPG_VERSION}" + } + } + } + stage ('Build image') { + steps { + sh """ + MAJ_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f1 -d'.') + echo \$MAJ_VER + MIN_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f2 -d'.') + echo \$MIN_VER + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-distribution-postgresql-\$MAJ_VER + sed -E "s/ENV PPG_VERSION (.+)/ENV PPG_VERSION ${params.PPG_VERSION}/" -i Dockerfile.aarch64 + sed -E "s/ENV PPG_REPO (.+)/ENV PPG_REPO ${params.PPG_REPO}/" -i Dockerfile.aarch64 + sed -E "s/ENV PPG_MAJOR_VERSION (.+)/ENV PPG_MAJOR_VERSION \$MAJ_VER/" -i Dockerfile.aarch64 + sed -E "s/ENV PPG_MINOR_VERSION (.+)/ENV PPG_MINOR_VERSION \$MIN_VER/" -i Dockerfile.aarch64 + docker build . -t percona-distribution-postgresql -f Dockerfile.aarch64 + """ + } + } + stage ('Push images to perconalab') { + when { + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + docker login -u '${USER}' -p '${PASS}' + MAJ_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f1 -d'.') + MIN_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f2 -d'.') + docker tag percona-distribution-postgresql perconalab/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 + docker tag percona-distribution-postgresql perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 + docker tag percona-distribution-postgresql perconalab/percona-distribution-postgresql:\$MAJ_VER-arm64 + docker push perconalab/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 + docker push perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 + docker push perconalab/percona-distribution-postgresql:\$MAJ_VER-arm64 + + docker manifest create --amend perconalab/percona-distribution-postgresql:${params.PPG_VERSION} \ + perconalab/percona-distribution-postgresql:${params.PPG_VERSION}-amd64 \ + perconalab/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 + docker manifest annotate perconalab/percona-distribution-postgresql:${params.PPG_VERSION} \ + perconalab/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-distribution-postgresql:${params.PPG_VERSION} \ + perconalab/percona-distribution-postgresql:${params.PPG_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-distribution-postgresql:${params.PPG_VERSION} + docker manifest push perconalab/percona-distribution-postgresql:${params.PPG_VERSION} + + docker manifest create --amend perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER \ + perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-amd64 \ + perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 + docker manifest annotate perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER \ + perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER \ + perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER + docker manifest push perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER + + docker manifest create --amend perconalab/percona-distribution-postgresql:\$MAJ_VER \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-amd64 \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-arm64 + docker manifest annotate perconalab/percona-distribution-postgresql:\$MAJ_VER \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-distribution-postgresql:\$MAJ_VER \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-distribution-postgresql:\$MAJ_VER + docker manifest push perconalab/percona-distribution-postgresql:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend perconalab/percona-distribution-postgresql:latest \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-amd64 \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-arm64 + docker manifest annotate perconalab/percona-distribution-postgresql:latest \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-distribution-postgresql:latest \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-distribution-postgresql:latest + docker manifest push perconalab/percona-distribution-postgresql:latest + fi + + """ + } + } + } + stage ('Push images to percona') { + when { + environment name: 'TARGET_REPO', value: 'DockerHub' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + docker login -u '${USER}' -p '${PASS}' + MAJ_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f1 -d'.') + MIN_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f2 -d'.') + docker tag percona-distribution-postgresql percona/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 + docker tag percona-distribution-postgresql percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 + docker tag percona-distribution-postgresql percona/percona-distribution-postgresql:\$MAJ_VER-arm64 + docker push percona/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 + docker push percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 + docker push percona/percona-distribution-postgresql:\$MAJ_VER-arm64 + + docker manifest create --amend percona/percona-distribution-postgresql:${params.PPG_VERSION} \ + percona/percona-distribution-postgresql:${params.PPG_VERSION}-amd64 \ + percona/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 + docker manifest annotate percona/percona-distribution-postgresql:${params.PPG_VERSION} \ + percona/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-distribution-postgresql:${params.PPG_VERSION} \ + percona/percona-distribution-postgresql:${params.PPG_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-distribution-postgresql:${params.PPG_VERSION} + docker manifest push percona/percona-distribution-postgresql:${params.PPG_VERSION} + + docker manifest create --amend percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER \ + percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-amd64 \ + percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 + docker manifest annotate percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER \ + percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER \ + percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER + docker manifest push percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER + + docker manifest create --amend percona/percona-distribution-postgresql:\$MAJ_VER \ + percona/percona-distribution-postgresql:\$MAJ_VER-amd64 \ + percona/percona-distribution-postgresql:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-distribution-postgresql:\$MAJ_VER \ + percona/percona-distribution-postgresql:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-distribution-postgresql:\$MAJ_VER \ + percona/percona-distribution-postgresql:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-distribution-postgresql:\$MAJ_VER + docker manifest push percona/percona-distribution-postgresql:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend percona/percona-distribution-postgresql:latest \ + percona/percona-distribution-postgresql:\$MAJ_VER-amd64 \ + percona/percona-distribution-postgresql:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-distribution-postgresql:latest \ + percona/percona-distribution-postgresql:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-distribution-postgresql:latest \ + percona/percona-distribution-postgresql:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-distribution-postgresql:latest + docker manifest push percona/percona-distribution-postgresql:latest + fi + """ + } + } + } + } + post { + always { + sh """ + sudo docker rmi -f \$(sudo docker images -q | uniq) || true + sudo rm -rf ${WORKSPACE}/* + """ + deleteDir() + } + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: Building of PPG ${PPG_VERSION} for ARM, repo ${PPG_REPO} succeed") + } + unstable { + slackNotify("#releases-ci", "#F6F930", "[${JOB_NAME}]: Building of PPG ${PPG_VERSION} fro ARM, repo ${PPG_REPO} unstable - [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: Building of PPG ${PPG_VERSION} for ARM, repo ${PPG_REPO} failed - [${BUILD_URL}]") + } + } +} diff --git a/ppg/ppg-docker-arm.yml b/ppg/ppg-docker-arm.yml new file mode 100644 index 0000000000..d789254aba --- /dev/null +++ b/ppg/ppg-docker-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-ppg-docker-arm + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ppg-docker-arm.groovy diff --git a/ppg/ppg-docker.groovy b/ppg/ppg-docker.groovy index 6af0fdcd97..179af39dbb 100644 --- a/ppg/ppg-docker.groovy +++ b/ppg/ppg-docker.groovy @@ -1,16 +1,17 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PPG_VERSION', defaultValue: '16.3-1', description: 'PPG version') choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') diff --git a/ppg/ppg-docker.yml b/ppg/ppg-docker.yml index 63c482693d..a70341aa47 100644 --- a/ppg/ppg-docker.yml +++ b/ppg/ppg-docker.yml @@ -1,5 +1,5 @@ - job: - name: ppg-docker + name: hetzner-ppg-docker project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/ppg-docker.groovy diff --git a/ppg/ppg-pgbackrest-docker-arm.groovy b/ppg/ppg-pgbackrest-docker-arm.groovy new file mode 100644 index 0000000000..6444735afd --- /dev/null +++ b/ppg/ppg-pgbackrest-docker-arm.groovy @@ -0,0 +1,183 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') + string(name: 'PGBACKREST_VERSION', defaultValue: '2.51-1', description: 'pgBackrest version') + choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') + string(name: 'PPG_VERSION', defaultValue: '17.0', description: 'PPG version') + choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') + choice(name: 'LATEST', choices: ['no','yes'], description: 'Tag image as latest') + } + options { + disableConcurrentBuilds() + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PPG_REPO}-${params.PPG_VERSION}" + } + } + } + stage ('Build image') { + steps { + sh """ + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-pgbackrest + sed -E "s/ENV PG_VERSION (.+)/ENV PG_VERSION ${params.PPG_VERSION}/" -i Dockerfile + sed -E "s/ENV PPG_REPO (.+)/ENV PPG_REPO ${params.PPG_REPO}/" -i Dockerfile + docker build . -t percona-pgbackrest + """ + } + } + stage ('Push image to aws ecr') { + when { + environment name: 'TARGET_REPO', value: 'AWS_ECR' + } + steps { + withCredentials([aws(accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: '8468e4e0-5371-4741-a9bb-7c143140acea', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) { + sh """ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + if [ -f "/usr/bin/yum" ] ; then sudo yum install -y unzip ; else sudo apt-get update && apt-get -y install unzip ; fi + unzip -o awscliv2.zip + sudo ./aws/install + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 + docker tag percona-pgbackrest public.ecr.aws/e7j3v3n0/percona-pgbackrest-build:percona-pgbackrest:${params.PGBACKREST_VERSION} + docker push public.ecr.aws/e7j3v3n0/percona-pgbackrest-build:percona-pgbackrest:${params.PGBACKREST_VERSION} + if [ ${params.LATEST} = "yes" ]; then + docker tag percona-pgbackrest public.ecr.aws/e7j3v3n0/percona-pgbackrest:latest + docker push public.ecr.aws/e7j3v3n0/percona-pgbackrest:latest + fi + """ + } + } + } + stage ('Push images to perconalab') { + when { + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + MAJ_VER=\$(echo ${params.PGBACKREST_VERSION} | cut -f1 -d'-') + docker login -u '${USER}' -p '${PASS}' + docker tag percona-pgbackrest perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 + docker push perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 + docker tag percona-pgbackrest perconalab/percona-pgbackrest:\$MAJ_VER-arm64 + docker push perconalab/percona-pgbackrest:\$MAJ_VER-arm64 + + docker manifest create --amend perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION} \ + perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 \ + perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 + docker manifest annotate perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION} \ + perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION} \ + perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION} + docker manifest push perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION} + + docker manifest create --amend perconalab/percona-pgbackrest:\$MAJ_VER \ + perconalab/percona-pgbackrest:\$MAJ_VER-amd64 \ + perconalab/percona-pgbackrest:\$MAJ_VER-arm64 + docker manifest annotate perconalab/percona-pgbackrest:\$MAJ_VER \ + perconalab/percona-pgbackrest:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-pgbackrest:\$MAJ_VER \ + perconalab/percona-pgbackrest:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-pgbackrest:\$MAJ_VER + docker manifest push perconalab/percona-pgbackrest:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend perconalab/percona-pgbackrest:latest \ + perconalab/percona-pgbackrest:\$MAJ_VER-amd64 \ + perconalab/percona-pgbackrest:\$MAJ_VER-arm64 + docker manifest annotate perconalab/percona-pgbackrest:latest \ + perconalab/percona-pgbackrest:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-pgbackrest:latest \ + perconalab/percona-pgbackrest:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-pgbackrest:latest + docker manifest push perconalab/percona-pgbackrest:latest + fi + + """ + } + } + } + stage ('Push images to official percona docker registry') { + when { + environment name: 'TARGET_REPO', value: 'DockerHub' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + MAJ_VER=\$(echo ${params.PGBACKREST_VERSION} | cut -f1 -d'-') + docker login -u '${USER}' -p '${PASS}' + docker tag percona-pgbackrest percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 + docker push percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 + docker tag percona-pgbackrest percona/percona-pgbackrest:\$MAJ_VER-arm64 + docker push percona/percona-pgbackrest:\$MAJ_VER-arm64 + + docker manifest create --amend percona/percona-pgbackrest:${params.PGBACKREST_VERSION} \ + percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 \ + percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 + docker manifest annotate percona/percona-pgbackrest:${params.PGBACKREST_VERSION} \ + percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-pgbackrest:${params.PGBACKREST_VERSION} \ + percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-pgbackrest:${params.PGBACKREST_VERSION} + docker manifest push percona/percona-pgbackrest:${params.PGBACKREST_VERSION} + + docker manifest create --amend percona/percona-pgbackrest:\$MAJ_VER \ + percona/percona-pgbackrest:\$MAJ_VER-amd64 \ + percona/percona-pgbackrest:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-pgbackrest:\$MAJ_VER \ + percona/percona-pgbackrest:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-pgbackrest:\$MAJ_VER \ + percona/percona-pgbackrest:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-pgbackrest:\$MAJ_VER + docker manifest push percona/percona-pgbackrest:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend percona/percona-pgbackrest:latest \ + percona/percona-pgbackrest:\$MAJ_VER-amd64 \ + percona/percona-pgbackrest:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-pgbackrest:latest \ + percona/percona-pgbackrest:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-pgbackrest:latest \ + percona/percona-pgbackrest:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-pgbackrest:latest + docker manifest push percona/percona-pgbackrest:latest + fi + """ + } + } + } + } + post { + always { + sh """ + sudo docker rmi -f \$(sudo docker images -q | uniq) || true + sudo rm -rf ${WORKSPACE}/* + """ + deleteDir() + } + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: Building of Percona pgBackrest ${PGBACKREST_VERSION} repo ${PPG_REPO} succeed") + } + unstable { + slackNotify("#releases-ci", "#F6F930", "[${JOB_NAME}]: Building of Percona pgBackrest ${PGBACKREST_VERSION} repo ${PPG_REPO} unstable - [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: Building of Percona pgBackrest ${PGBACKREST_VERSION} repo ${PPG_REPO} failed - [${BUILD_URL}]") + } + } +} diff --git a/ppg/ppg-pgbackrest-docker-arm.yml b/ppg/ppg-pgbackrest-docker-arm.yml new file mode 100644 index 0000000000..577b1b07e2 --- /dev/null +++ b/ppg/ppg-pgbackrest-docker-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-percona-pgbackrest-docker-arm + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ppg-pgbackrest-docker-arm.groovy diff --git a/ppg/ppg-pgbackrest-docker.groovy b/ppg/ppg-pgbackrest-docker.groovy index eef8f70697..f8597bd4df 100644 --- a/ppg/ppg-pgbackrest-docker.groovy +++ b/ppg/ppg-pgbackrest-docker.groovy @@ -1,17 +1,18 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { - string(name: 'PGBACKREST_VERSION', defaultValue: '2.51-1', description: 'pgBackrest version') + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') + string(name: 'PGBACKREST_VERSION', defaultValue: '2.51-1', description: 'pgBackrest version') choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PPG_VERSION', defaultValue: '17.0', description: 'PPG version') choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') @@ -35,7 +36,7 @@ pipeline { cd percona-docker/percona-pgbackrest sed -E "s/ENV PG_VERSION (.+)/ENV PG_VERSION ${params.PPG_VERSION}/" -i Dockerfile sed -E "s/ENV PPG_REPO (.+)/ENV PPG_REPO ${params.PPG_REPO}/" -i Dockerfile - docker build . -t percona-pgbackrest:${params.PGBACKREST_VERSION} + docker build . -t percona-pgbackrest """ } } @@ -48,10 +49,10 @@ pipeline { wget https://raw.githubusercontent.com/aquasecurity/trivy/v\${TRIVY_VERSION}/contrib/junit.tpl if [ ${params.PPG_REPO} = "release" ]; then /usr/local/bin/trivy -q image --format template --template @junit.tpl -o trivy-hight-junit.xml \ - --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-pgbackrest:${params.PGBACKREST_VERSION} + --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-pgbackrest else /usr/local/bin/trivy -q image --format template --template @junit.tpl -o trivy-hight-junit.xml \ - --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-pgbackrest:${params.PGBACKREST_VERSION} + --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-pgbackrest fi """ } @@ -73,10 +74,10 @@ pipeline { unzip -o awscliv2.zip sudo ./aws/install aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} public.ecr.aws/e7j3v3n0/percona-pgbackrest-build:percona-pgbackrest:${params.PGBACKREST_VERSION} + docker tag percona-pgbackrest public.ecr.aws/e7j3v3n0/percona-pgbackrest-build:percona-pgbackrest:${params.PGBACKREST_VERSION} docker push public.ecr.aws/e7j3v3n0/percona-pgbackrest-build:percona-pgbackrest:${params.PGBACKREST_VERSION} if [ ${params.LATEST} = "yes" ]; then - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} public.ecr.aws/e7j3v3n0/percona-pgbackrest:latest + docker tag percona-pgbackrest public.ecr.aws/e7j3v3n0/percona-pgbackrest:latest docker push public.ecr.aws/e7j3v3n0/percona-pgbackrest:latest fi """ @@ -92,12 +93,12 @@ pipeline { sh """ MAJ_VER=\$(echo ${params.PGBACKREST_VERSION} | cut -f1 -d'-') docker login -u '${USER}' -p '${PASS}' - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 + docker tag percona-pgbackrest perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 docker push perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} perconalab/percona-pgbackrest:\$MAJ_VER-amd64 + docker tag percona-pgbackrest perconalab/percona-pgbackrest:\$MAJ_VER-amd64 docker push perconalab/percona-pgbackrest:\$MAJ_VER-amd64 if [ ${params.LATEST} = "yes" ]; then - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} perconalab/percona-pgbackrest:latest + docker tag percona-pgbackrest perconalab/percona-pgbackrest:latest docker push perconalab/percona-pgbackrest:latest fi """ @@ -113,12 +114,12 @@ pipeline { sh """ MAJ_VER=\$(echo ${params.PGBACKREST_VERSION} | cut -f1 -d'-') docker login -u '${USER}' -p '${PASS}' - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 + docker tag percona-pgbackrest percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 docker push percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} percona/percona-pgbackrest:\$MAJ_VER-amd64 + docker tag percona-pgbackrest percona/percona-pgbackrest:\$MAJ_VER-amd64 docker push percona/percona-pgbackrest:\$MAJ_VER-amd64 if [ ${params.LATEST} = "yes" ]; then - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} percona/percona-pgbackrest:latest + docker tag percona-pgbackrest percona/percona-pgbackrest:latest docker push percona/percona-pgbackrest:latest fi """ diff --git a/ppg/ppg-pgbackrest-docker.yml b/ppg/ppg-pgbackrest-docker.yml index c584383a39..57bb7f9efd 100644 --- a/ppg/ppg-pgbackrest-docker.yml +++ b/ppg/ppg-pgbackrest-docker.yml @@ -1,5 +1,5 @@ - job: - name: percona-pgbackrest-docker + name: hetzner-percona-pgbackrest-docker project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/ppg-pgbackrest-docker.groovy diff --git a/ppg/ppg-pgbouncer-docker-arm.groovy b/ppg/ppg-pgbouncer-docker-arm.groovy new file mode 100644 index 0000000000..7f2bd68b0a --- /dev/null +++ b/ppg/ppg-pgbouncer-docker-arm.groovy @@ -0,0 +1,182 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') + string(name: 'PGBOUNCER_VERSION', defaultValue: '1.22.1-1', description: 'pgBouncer version') + choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') + string(name: 'PPG_VERSION', defaultValue: '17.0', description: 'PPG version') + choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') + choice(name: 'LATEST', choices: ['no','yes'], description: 'Tag image as latest') + } + options { + disableConcurrentBuilds() + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PPG_REPO}-${params.PPG_VERSION}" + } + } + } + stage ('Build image') { + steps { + sh """ + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-pgbouncer + sed -E "s/ENV PG_VERSION (.+)/ENV PG_VERSION ${params.PPG_VERSION}/" -i Dockerfile + sed -E "s/ENV PPG_REPO (.+)/ENV PPG_REPO ${params.PPG_REPO}/" -i Dockerfile + docker build . -t percona-pgbouncer + """ + } + } + stage ('Push image to aws ecr') { + when { + environment name: 'TARGET_REPO', value: 'AWS_ECR' + } + steps { + withCredentials([aws(accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: '8468e4e0-5371-4741-a9bb-7c143140acea', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) { + sh """ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + if [ -f "/usr/bin/yum" ] ; then sudo yum install -y unzip ; else sudo apt-get update && apt-get -y install unzip ; fi + unzip -o awscliv2.zip + sudo ./aws/install + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 + docker tag percona-pgbouncer public.ecr.aws/e7j3v3n0/percona-pgbouncer-build:percona-pgbouncer:${params.PGBOUNCER_VERSION} + docker push public.ecr.aws/e7j3v3n0/percona-pgbouncer-build:percona-pgbouncer:${params.PGBOUNCER_VERSION} + if [ ${params.LATEST} = "yes" ]; then + docker tag percona-pgbouncer public.ecr.aws/e7j3v3n0/percona-pgbouncer:latest + docker push public.ecr.aws/e7j3v3n0/percona-pgbouncer:latest + fi + """ + } + } + } + stage ('Push images to perconalab') { + when { + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + MAJ_VER=\$(echo ${params.PGBOUNCER_VERSION} | cut -f1 -d'-') + docker login -u '${USER}' -p '${PASS}' + docker tag percona-pgbouncer perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 + docker push perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 + docker tag percona-pgbouncer perconalab/percona-pgbouncer:\$MAJ_VER-arm64 + docker push perconalab/percona-pgbouncer:\$MAJ_VER-arm64 + + docker manifest create --amend perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION} \ + perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 \ + perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 + docker manifest annotate perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION} \ + perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION} \ + perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION} + docker manifest push perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION} + + docker manifest create --amend perconalab/percona-pgbouncer:\$MAJ_VER \ + perconalab/percona-pgbouncer:\$MAJ_VER-amd64 \ + perconalab/percona-pgbouncer:\$MAJ_VER-arm64 + docker manifest annotate perconalab/percona-pgbouncer:\$MAJ_VER \ + perconalab/percona-pgbouncer:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-pgbouncer:\$MAJ_VER \ + perconalab/percona-pgbouncer:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-pgbouncer:\$MAJ_VER + docker manifest push perconalab/percona-pgbouncer:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend perconalab/percona-pgbouncer:latest \ + perconalab/percona-pgbouncer:\$MAJ_VER-amd64 \ + perconalab/percona-pgbouncer:\$MAJ_VER-arm64 + docker manifest annotate perconalab/percona-pgbouncer:latest \ + perconalab/percona-pgbouncer:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-pgbouncer:latest \ + perconalab/percona-pgbouncer:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-pgbouncer:latest + docker manifest push perconalab/percona-pgbouncer:latest + fi + """ + } + } + } + stage ('Push images to official percona docker registry') { + when { + environment name: 'TARGET_REPO', value: 'DockerHub' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + MAJ_VER=\$(echo ${params.PGBOUNCER_VERSION} | cut -f1 -d'-') + docker login -u '${USER}' -p '${PASS}' + docker tag percona-pgbouncer percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 + docker push percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 + docker tag percona-pgbouncer percona/percona-pgbouncer:\$MAJ_VER-arm64 + docker push percona/percona-pgbouncer:\$MAJ_VER-arm64 + + docker manifest create --amend percona/percona-pgbouncer:${params.PGBOUNCER_VERSION} \ + percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 \ + percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 + docker manifest annotate percona/percona-pgbouncer:${params.PGBOUNCER_VERSION} \ + percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-pgbouncer:${params.PGBOUNCER_VERSION} \ + percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-pgbouncer:${params.PGBOUNCER_VERSION} + docker manifest push percona/percona-pgbouncer:${params.PGBOUNCER_VERSION} + + docker manifest create --amend percona/percona-pgbouncer:\$MAJ_VER \ + percona/percona-pgbouncer:\$MAJ_VER-amd64 \ + percona/percona-pgbouncer:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-pgbouncer:\$MAJ_VER \ + percona/percona-pgbouncer:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-pgbouncer:\$MAJ_VER \ + percona/percona-pgbouncer:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-pgbouncer:\$MAJ_VER + docker manifest push percona/percona-pgbouncer:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend percona/percona-pgbouncer:latest \ + percona/percona-pgbouncer:\$MAJ_VER-amd64 \ + percona/percona-pgbouncer:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-pgbouncer:latest \ + percona/percona-pgbouncer:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-pgbouncer:latest \ + percona/percona-pgbouncer:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-pgbouncer:latest + docker manifest push percona/percona-pgbouncer:latest + fi + """ + } + } + } + } + post { + always { + sh """ + sudo docker rmi -f \$(sudo docker images -q | uniq) || true + sudo rm -rf ${WORKSPACE}/* + """ + deleteDir() + } + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: Building of Percona pgBouncer ${PGBOUNCER_VERSION} repo ${PPG_REPO} succeed") + } + unstable { + slackNotify("#releases-ci", "#F6F930", "[${JOB_NAME}]: Building of Percona pgBouncer ${PGBOUNCER_VERSION} repo ${PPG_REPO} unstable - [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: Building of Percona pgBouncer ${PGBOUNCER_VERSION} repo ${PPG_REPO} failed - [${BUILD_URL}]") + } + } +} diff --git a/ppg/ppg-pgbouncer-docker-arm.yml b/ppg/ppg-pgbouncer-docker-arm.yml new file mode 100644 index 0000000000..3e99e9246d --- /dev/null +++ b/ppg/ppg-pgbouncer-docker-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-percona-pgbouncer-docker-arm + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ppg-pgbouncer-docker-arm.groovy diff --git a/ppg/ppg-pgbouncer-docker.groovy b/ppg/ppg-pgbouncer-docker.groovy index dfcc2097dd..f18dece44b 100644 --- a/ppg/ppg-pgbouncer-docker.groovy +++ b/ppg/ppg-pgbouncer-docker.groovy @@ -1,16 +1,17 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') string(name: 'PGBOUNCER_VERSION', defaultValue: '1.22.1-1', description: 'pgBouncer version') choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PPG_VERSION', defaultValue: '17.0', description: 'PPG version') @@ -35,7 +36,7 @@ pipeline { cd percona-docker/percona-pgbouncer sed -E "s/ENV PG_VERSION (.+)/ENV PG_VERSION ${params.PPG_VERSION}/" -i Dockerfile sed -E "s/ENV PPG_REPO (.+)/ENV PPG_REPO ${params.PPG_REPO}/" -i Dockerfile - docker build . -t percona-pgbouncer:${params.PGBOUNCER_VERSION} + docker build . -t percona-pgbouncer """ } } @@ -48,10 +49,10 @@ pipeline { wget https://raw.githubusercontent.com/aquasecurity/trivy/v\${TRIVY_VERSION}/contrib/junit.tpl if [ ${params.PPG_REPO} = "release" ]; then /usr/local/bin/trivy -q image --format template --template @junit.tpl -o trivy-hight-junit.xml \ - --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-pgbouncer:${params.PGBOUNCER_VERSION} + --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-pgbouncer else /usr/local/bin/trivy -q image --format template --template @junit.tpl -o trivy-hight-junit.xml \ - --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-pgbouncer:${params.PGBOUNCER_VERSION} + --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-pgbouncer fi """ } @@ -73,10 +74,10 @@ pipeline { unzip -o awscliv2.zip sudo ./aws/install aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} public.ecr.aws/e7j3v3n0/percona-pgbouncer-build:percona-pgbouncer:${params.PGBOUNCER_VERSION} + docker tag percona-pgbouncer public.ecr.aws/e7j3v3n0/percona-pgbouncer-build:percona-pgbouncer:${params.PGBOUNCER_VERSION} docker push public.ecr.aws/e7j3v3n0/percona-pgbouncer-build:percona-pgbouncer:${params.PGBOUNCER_VERSION} if [ ${params.LATEST} = "yes" ]; then - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} public.ecr.aws/e7j3v3n0/percona-pgbouncer:latest + docker tag percona-pgbouncer public.ecr.aws/e7j3v3n0/percona-pgbouncer:latest docker push public.ecr.aws/e7j3v3n0/percona-pgbouncer:latest fi """ @@ -92,12 +93,12 @@ pipeline { sh """ MAJ_VER=\$(echo ${params.PGBOUNCER_VERSION} | cut -f1 -d'-') docker login -u '${USER}' -p '${PASS}' - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 + docker tag percona-pgbouncer perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 docker push perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} perconalab/percona-pgbouncer:\$MAJ_VER-amd64 + docker tag percona-pgbouncer perconalab/percona-pgbouncer:\$MAJ_VER-amd64 docker push perconalab/percona-pgbouncer:\$MAJ_VER-amd64 if [ ${params.LATEST} = "yes" ]; then - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} perconalab/percona-pgbouncer:latest + docker tag percona-pgbouncer perconalab/percona-pgbouncer:latest docker push perconalab/percona-pgbouncer:latest fi """ @@ -113,12 +114,12 @@ pipeline { sh """ MAJ_VER=\$(echo ${params.PGBOUNCER_VERSION} | cut -f1 -d'-') docker login -u '${USER}' -p '${PASS}' - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 + docker tag percona-pgbouncer percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 docker push percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} percona/percona-pgbouncer:\$MAJ_VER-amd64 + docker tag percona-pgbouncer percona/percona-pgbouncer:\$MAJ_VER-amd64 docker push percona/percona-pgbouncer:\$MAJ_VER-amd64 if [ ${params.LATEST} = "yes" ]; then - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} percona/percona-pgbouncer:latest + docker tag percona-pgbouncer percona/percona-pgbouncer:latest docker push percona/percona-pgbouncer:latest fi """ diff --git a/ppg/ppg-pgbouncer-docker.yml b/ppg/ppg-pgbouncer-docker.yml index 1d988e6831..e34912d882 100644 --- a/ppg/ppg-pgbouncer-docker.yml +++ b/ppg/ppg-pgbouncer-docker.yml @@ -1,5 +1,5 @@ - job: - name: percona-pgbouncer-docker + name: hetzner-percona-pgbouncer-docker project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/ppg-pgbouncer-docker.groovy diff --git a/ppg/ppg-server-arm.groovy b/ppg/ppg-server-arm.groovy new file mode 100644 index 0000000000..e6d89101b5 --- /dev/null +++ b/ppg/ppg-server-arm.groovy @@ -0,0 +1,250 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/ppg-server/ppg-server_builder.sh -O ppg-server_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${PPG_REPO} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for ppg-server repository', + name: 'GIT_REPO') + string( + defaultValue: '14.4', + description: 'Tag/Branch for ppg-server repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-14.4', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PPG_SERVER source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/ppg-server.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/ppg-server.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PPG-SERVER generic source packages') { + parallel { + stage('Build PPG-SERVER generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PPG-SERVER generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PPG-SERVER RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/ppg-server-arm.yml b/ppg/ppg-server-arm.yml new file mode 100644 index 0000000000..bd4c2daa77 --- /dev/null +++ b/ppg/ppg-server-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-ppg-server-arm-autobuild-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ppg-server-arm.groovy + diff --git a/ppg/ppg-server-ha-arm.groovy b/ppg/ppg-server-ha-arm.groovy new file mode 100644 index 0000000000..c89ba9b6ad --- /dev/null +++ b/ppg/ppg-server-ha-arm.groovy @@ -0,0 +1,250 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/ppg-server-ha/ppg-server-ha_builder.sh -O ppg-server-ha_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppg-server-ha_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ppg-server-ha_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${PPG_REPO} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for ppg-server-ha repository', + name: 'GIT_REPO') + string( + defaultValue: '14.4', + description: 'Tag/Branch for ppg-server-ha repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-14.4', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PPG_SERVER_HA source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/ppg-server-ha.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/ppg-server-ha.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PPG-SERVER_HA generic source packages') { + parallel { + stage('Build PPG-SERVER_HA generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PPG-SERVER_HA generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PPG-SERVER-HA RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/ppg-server-ha-arm.yml b/ppg/ppg-server-ha-arm.yml new file mode 100644 index 0000000000..58122159ef --- /dev/null +++ b/ppg/ppg-server-ha-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-ppg-server-ha-arm-autobuild-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ppg-server-ha-arm.groovy + diff --git a/ppg/ppg-server-ha.groovy b/ppg/ppg-server-ha.groovy index e097648111..cd33529b51 100644 --- a/ppg/ppg-server-ha.groovy +++ b/ppg/ppg-server-ha.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for ppg-server-ha repository', @@ -80,131 +84,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PPG-SERVER_HA generic source packages') { parallel { stage('Build PPG-SERVER_HA generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PPG-SERVER_HA generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PPG-SERVER-HA RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -212,14 +307,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/ppg-server-ha.yml b/ppg/ppg-server-ha.yml index 64e0bd6938..2d74c54f08 100644 --- a/ppg/ppg-server-ha.yml +++ b/ppg/ppg-server-ha.yml @@ -1,5 +1,5 @@ - job: - name: ppg-server-ha-autobuild-RELEASE + name: hetzner-ppg-server-ha-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/ppg-server-ha.groovy diff --git a/ppg/ppg-server.groovy b/ppg/ppg-server.groovy index 82340a9072..6534d0c9a4 100644 --- a/ppg/ppg-server.groovy +++ b/ppg/ppg-server.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for ppg-server repository', @@ -80,131 +84,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PPG-SERVER generic source packages') { parallel { stage('Build PPG-SERVER generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PPG-SERVER generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PPG-SERVER RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -212,14 +307,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/ppg-server.yml b/ppg/ppg-server.yml index 5d334a1cf2..8dbf177b84 100644 --- a/ppg/ppg-server.yml +++ b/ppg/ppg-server.yml @@ -1,5 +1,5 @@ - job: - name: ppg-server-autobuild-RELEASE + name: hetzner-ppg-server-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/ppg-server.groovy diff --git a/ppg/ppg_release.groovy b/ppg/ppg_release.groovy index 3449f0a87f..a75229ac41 100644 --- a/ppg/ppg_release.groovy +++ b/ppg/ppg_release.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@ENG-7_postgresql_rel', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -13,9 +13,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) text(name: 'PACKAGES', defaultValue: '', description: 'put all pathes to all packages') string(name: 'repo_version', defaultValue: '', description: 'Repository Version i.e ppg-15.3 or ppg-15') choice(name: 'component', choices: ['release', 'testing', 'experimental', 'laboratory'], description: 'Component') @@ -62,12 +66,12 @@ pipeline { } stage('Push to public repository') { steps { - sync2ProdPPG(repo_version, component) + sync2ProdPPG(params.CLOUD, repo_version, component) } } stage('Push to WEB') { steps { - sync2web(repo_version, component) + sync2web(params.CLOUD, repo_version, component) } } } diff --git a/ppg/ppg_release.yml b/ppg/ppg_release.yml index 246edb818a..5fcf30c4f6 100644 --- a/ppg/ppg_release.yml +++ b/ppg/ppg_release.yml @@ -1,5 +1,5 @@ - job: - name: Percona-PostgreSQL-Distribution-RELEASE + name: hetzner-Percona-PostgreSQL-Distribution-RELEASE project-type: pipeline description: | Do not edit this job through the web! diff --git a/ppg/pysyncobj-arm.groovy b/ppg/pysyncobj-arm.groovy new file mode 100644 index 0000000000..97ac3d2158 --- /dev/null +++ b/ppg/pysyncobj-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pysyncobj/pysyncobj_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pysyncobj repository', + name: 'GIT_REPO') + string( + defaultValue: '0.3.10', + description: 'Tag/Branch for pysyncobj repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.3', + description: 'Tag/Branch for postgresql packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.3', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PYSYNCOBJ source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pysyncobj.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pysyncobj.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PYSYNCOBJ generic source packages') { + parallel { + stage('Build PYSYNCOBJ generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PYSYNCOBJ generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PYSYNCOBJ RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pysyncobj-arm.yml b/ppg/pysyncobj-arm.yml new file mode 100644 index 0000000000..20b6f7db9e --- /dev/null +++ b/ppg/pysyncobj-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pysyncobj-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pysyncobj-arm.groovy + diff --git a/ppg/pysyncobj.groovy b/ppg/pysyncobj.groovy index 96d434181e..819d25da28 100644 --- a/ppg/pysyncobj.groovy +++ b/ppg/pysyncobj.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pysyncobj repository', @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PYSYNCOBJ generic source packages') { parallel { stage('Build PYSYNCOBJ generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PYSYNCOBJ generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,93 +126,93 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +220,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pysyncobj.yml b/ppg/pysyncobj.yml index ce7e345525..012934ebd0 100644 --- a/ppg/pysyncobj.yml +++ b/ppg/pysyncobj.yml @@ -1,5 +1,5 @@ - job: - name: pysyncobj-RELEASE + name: hetzner-pysyncobj-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pysyncobj.groovy diff --git a/ppg/timescaledb-arm.groovy b/ppg/timescaledb-arm.groovy new file mode 100644 index 0000000000..ab73f02fbe --- /dev/null +++ b/ppg/timescaledb-arm.groovy @@ -0,0 +1,253 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/timescaledb/timescaledb_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for timescaledb repository', + name: 'GIT_REPO') + string( + defaultValue: '2.15.2', + description: 'Branch/Tag of the timescaledb', + name: 'PG_BRANCH') + string( + defaultValue: '16.3', + description: 'Tag/Branch for packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16-extras', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create timescaledb source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/timescaledb.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/timescaledb.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build timescaledb generic source packages') { + parallel { + stage('Build timescaledb generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build timescaledb generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build timescaledb RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/timescaledb-arm.yml b/ppg/timescaledb-arm.yml new file mode 100644 index 0000000000..52bff7a322 --- /dev/null +++ b/ppg/timescaledb-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-timescaledb-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/timescaledb-arm.groovy diff --git a/ppg/timescaledb.groovy b/ppg/timescaledb.groovy index 6b53b41c09..02b2a17e6c 100644 --- a/ppg/timescaledb.groovy +++ b/ppg/timescaledb.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for timescaledb repository', @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build timescaledb generic source packages') { parallel { stage('Build timescaledb generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build timescaledb generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,93 +126,93 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +220,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/timescaledb.yml b/ppg/timescaledb.yml index 22fb8cde7b..47970dff7a 100644 --- a/ppg/timescaledb.yml +++ b/ppg/timescaledb.yml @@ -1,5 +1,5 @@ - job: - name: timescaledb-RELEASE + name: hetzner-timescaledb-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/timescaledb.groovy diff --git a/ppg/wal2json-arm.groovy b/ppg/wal2json-arm.groovy new file mode 100644 index 0000000000..f67416bf28 --- /dev/null +++ b/ppg/wal2json-arm.groovy @@ -0,0 +1,253 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/wal2json/wal2json_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for wal2json repository', + name: 'GIT_REPO') + string( + defaultValue: '2.5', + description: 'General version of the product', + name: 'PG_BRANCH') + string( + defaultValue: '16.1', + description: 'Tag/Branch for wal2json repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.1', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create WAL2JSON source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/wal2json.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/wal2json.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build WAL2JSON generic source packages') { + parallel { + stage('Build WAL2JSON generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build wal2json generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build WAL2JSON RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/wal2json-arm.yml b/ppg/wal2json-arm.yml new file mode 100644 index 0000000000..3e345ed16b --- /dev/null +++ b/ppg/wal2json-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-wal2json-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/wal2json-arm.groovy \ No newline at end of file diff --git a/ppg/wal2json.groovy b/ppg/wal2json.groovy index 4367bb2fba..0aeff2adb6 100644 --- a/ppg/wal2json.groovy +++ b/ppg/wal2json.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for wal2json repository', @@ -84,131 +88,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build wal2json generic source packages') { parallel { stage('Build wal2json generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build wal2json generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build wal2json RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +311,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/wal2json.yml b/ppg/wal2json.yml index c0b09523a5..c76eff220e 100644 --- a/ppg/wal2json.yml +++ b/ppg/wal2json.yml @@ -1,5 +1,5 @@ - job: - name: wal2json-RELEASE + name: hetzner-wal2json-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/wal2json.groovy diff --git a/ppg/ydiff-arm.groovy b/ppg/ydiff-arm.groovy new file mode 100644 index 0000000000..753cd5eada --- /dev/null +++ b/ppg/ydiff-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/ydiff/ydiff_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for ydiff repository', + name: 'GIT_REPO') + string( + defaultValue: '1.2', + description: 'Tag/Branch for ydiff repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.3', + description: 'Tag/Branch for postgresql packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.3', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create YDIFF source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/ydiff.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/ydiff.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build YDIFF generic source packages') { + parallel { + stage('Build YDIFF generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build YDIFF generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build YDIFF RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/ydiff-arm.yml b/ppg/ydiff-arm.yml new file mode 100644 index 0000000000..790ef63f45 --- /dev/null +++ b/ppg/ydiff-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-ydiff-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ydiff-arm.groovy + diff --git a/ppg/ydiff.groovy b/ppg/ydiff.groovy index 759bfa22c3..9b5999d6b6 100644 --- a/ppg/ydiff.groovy +++ b/ppg/ydiff.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for ydiff repository', @@ -84,131 +88,157 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build YDIFF generic source packages') { parallel { stage('Build YDIFF generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build YDIFF generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build YDIFF RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +246,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/ydiff.yml b/ppg/ydiff.yml index c32229aac9..6074e5d0e5 100644 --- a/ppg/ydiff.yml +++ b/ppg/ydiff.yml @@ -1,5 +1,5 @@ - job: - name: ydiff-RELEASE + name: hetzner-ydiff-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/ydiff.groovy diff --git a/proxysql/proxysql.groovy b/proxysql/proxysql.groovy index 388bbd908a..0c9b852519 100644 --- a/proxysql/proxysql.groovy +++ b/proxysql/proxysql.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -30,9 +30,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/proxysql-packaging.git', description: 'URL for proxysql packaging repository', @@ -101,36 +105,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PROXYSQL generic source packages') { parallel { stage('Build PROXYSQL generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PROXYSQL generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -139,229 +143,298 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Oracle Linux 8 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } stage('Ubuntu Focal tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } + stage('Ubuntu Noble tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_tarball=1") + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') + } + } } } stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PROXYSQL_DEST_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PROXYSQL_DEST_REPO, COMPONENT) } } + stage('Build docker containers') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1200 + ''' + unstash 'uploadPath' + sh ''' + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + sudo systemctl restart docker + git clone https://github.com/percona/percona-docker + cd percona-docker/proxysql + sed -i "s/ENV PROXYSQL_VERSION.*/ENV PROXYSQL_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/enable proxysql testing/enable proxysql ${COMPONENT}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t perconalab/proxysql2:${VERSION}-${RPM_RELEASE} . + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/proxysql2:${VERSION}-${RPM_RELEASE} + + PROXYSQL_MAJOR_VERSION=$(echo $VERSION | cut -d'.' -f1) + PROXYSQL_MINOR_VERSION=$(echo $VERSION | cut -d'.' -f2) + PROXYSQL_PATCH_VERSION=$(echo $VERSION | cut -d'.' -f3) + + docker tag perconalab/proxysql2:${VERSION}-${RPM_RELEASE} perconalab/proxysql2:${PROXYSQL_MAJOR_VERSION}.${PROXYSQL_MINOR_VERSION}.${PROXYSQL_PATCH_VERSION} + docker tag perconalab/proxysql2:${VERSION}-${RPM_RELEASE} perconalab/proxysql2:${PROXYSQL_MAJOR_VERSION}.${PROXYSQL_MINOR_VERSION} + docker tag perconalab/proxysql2:${VERSION}-${RPM_RELEASE} perconalab/proxysql2:${PROXYSQL_MAJOR_VERSION} + + sudo docker push perconalab/proxysql2:${PROXYSQL_MAJOR_VERSION}.${PROXYSQL_MINOR_VERSION}.${PROXYSQL_PATCH_VERSION} + sudo docker push perconalab/proxysql2:${PROXYSQL_MAJOR_VERSION}.${PROXYSQL_MINOR_VERSION} + sudo docker push perconalab/proxysql2:${PROXYSQL_MAJOR_VERSION} + ''' + } + } + } + } } post { success { diff --git a/proxysql/proxysql.yml b/proxysql/proxysql.yml index 0db3f93ef5..15508eadaa 100644 --- a/proxysql/proxysql.yml +++ b/proxysql/proxysql.yml @@ -1,5 +1,5 @@ - job: - name: proxysql-RELEASE + name: hetzner-proxysql-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: proxysql/proxysql.groovy diff --git a/ps/jenkins/jemalloc.groovy b/ps/jenkins/jemalloc.groovy index 84386328a2..77d80f9dd4 100644 --- a/ps/jenkins/jemalloc.groovy +++ b/ps/jenkins/jemalloc.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -166,9 +166,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) /* string( defaultValue: 'https://github.com/percona-lab/jemalloc-packaging.git', @@ -226,15 +230,15 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build jemalloc packages') { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -250,15 +254,15 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -271,15 +275,15 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -292,15 +296,15 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -313,15 +317,15 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -333,13 +337,13 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -351,13 +355,13 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye (11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -369,13 +373,13 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye (11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -387,13 +391,13 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy (22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -405,13 +409,13 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy (22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -423,13 +427,13 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm (12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -441,13 +445,13 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm (12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -459,13 +463,13 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble (24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -477,13 +481,13 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble (24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -495,8 +499,8 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } diff --git a/ps/jenkins/jemalloc.yml b/ps/jenkins/jemalloc.yml index 1650207e9c..950a67f390 100644 --- a/ps/jenkins/jemalloc.yml +++ b/ps/jenkins/jemalloc.yml @@ -1,5 +1,5 @@ - job: - name: jemalloc-RELEASE + name: hetzner-jemalloc-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/jemalloc.groovy diff --git a/ps/jenkins/mysql-shell.groovy b/ps/jenkins/mysql-shell.groovy index 1c827f30ee..06b7d885fe 100644 --- a/ps/jenkins/mysql-shell.groovy +++ b/ps/jenkins/mysql-shell.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -28,9 +28,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/mysql-shell-packaging.git', description: 'URL for mysql-shell packaging repository', @@ -98,36 +102,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build MYSQL-SHELL generic source packages') { parallel { stage('Build MYSQL-SHELL generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build MYSQL-SHELL generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -137,97 +141,97 @@ pipeline { /* stage('Centos 7') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } */ stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { PS_MAJOR_RELEASE = sh(returnStdout: true, script: ''' echo ${PS_BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}' ''').trim() if ("${PS_MAJOR_RELEASE}" == "80") { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } else { echo "The step is skipped." } @@ -236,83 +240,83 @@ pipeline { } stage('Ubuntu Jammy (22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy (22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble (24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble (24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye (11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye (11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { PS_MAJOR_RELEASE = sh(returnStdout: true, script: ''' echo ${PS_BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}' ''').trim() if ("${PS_MAJOR_RELEASE}" == "80") { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } else { echo "The step is skipped." } @@ -321,67 +325,67 @@ pipeline { } stage('Debian Bookworm (12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm (12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } stage('Oracle Linux 9 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } stage('Debian Bullseye (11) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } @@ -396,7 +400,7 @@ pipeline { stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild('mysql-shell', COMPONENT) + sync2ProdAutoBuild(params.CLOUD, 'mysql-shell', COMPONENT) } } @@ -404,9 +408,6 @@ pipeline { post { success { // slackNotify("", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") - script { - currentBuild.description = "Built on ${SHELL_BRANCH}, path to packages: experimental/${AWS_STASH_PATH}" - } deleteDir() } failure { @@ -417,6 +418,9 @@ pipeline { sh ''' sudo rm -rf ./* ''' + script { + currentBuild.description = "Built on SHELL ${SHELL_BRANCH}, PS ${PS_BRANCH} path to packages: experimental/${AWS_STASH_PATH}" + } deleteDir() } } diff --git a/ps/jenkins/mysql-shell.yml b/ps/jenkins/mysql-shell.yml index 1789900126..8167530dcd 100644 --- a/ps/jenkins/mysql-shell.yml +++ b/ps/jenkins/mysql-shell.yml @@ -1,5 +1,5 @@ - job: - name: mysql-shell-8.0-RELEASE + name: hetzner-mysql-shell-8.0-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/mysql-shell.groovy diff --git a/ps/jenkins/percona-server-for-mysql-5.7.groovy b/ps/jenkins/percona-server-for-mysql-5.7.groovy index e83ed7c1e3..d0779c7669 100644 --- a/ps/jenkins/percona-server-for-mysql-5.7.groovy +++ b/ps/jenkins/percona-server-for-mysql-5.7.groovy @@ -1,24 +1,34 @@ /* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ void installCli(String PLATFORM) { sh """ - set -o xtrace - if [ -d aws ]; then - rm -rf aws - fi - if [ ${PLATFORM} = "deb" ]; then - sudo apt-get update - sudo apt-get -y install wget curl unzip - elif [ ${PLATFORM} = "rpm" ]; then - sudo yum -y install wget curl unzip - fi - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip - unzip awscliv2.zip - sudo ./aws/install || true + if [ \${CLOUD} = "AWS" ]; then + set -o xtrace + if [ -d aws ]; then + rm -rf aws + fi + if [ ${PLATFORM} = "deb" ]; then + sudo apt-get update + sudo apt-get -y install wget curl unzip + elif [ ${PLATFORM} = "rpm" ]; then + export RHVER=\$(rpm --eval %rhel) + if [ \${RHVER} = "7" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* || true + sudo sed -i 's|#\\s*baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true + if [ -e "/etc/yum.repos.d/CentOS-SCLo-scl.repo" ]; then + cat /etc/yum.repos.d/CentOS-SCLo-scl.repo + fi + fi + sudo yum -y install wget curl unzip + fi + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip + unzip awscliv2.zip + sudo ./aws/install || true + fi """ } @@ -43,16 +53,28 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { mkdir -p test wget --header="Authorization: token ${TOKEN}" --header="Accept: application/vnd.github.v3.raw" -O ps_builder.sh \$(echo ${GIT_REPO} | sed -re 's|github.com|api.github.com/repos|; s|\\.git\$||')/contents/build-ps/percona-server-5.7_builder.sh?ref=${BRANCH} sed -i "s|git clone \\\"\\\$REPO\\\"|git clone \$(echo ${GIT_REPO}| sed -re 's|github.com|${TOKEN}@github.com|') percona-server|g" ps_builder.sh - grep "git clone" ps_builder.sh pwd -P export build_dir=\$(pwd -P) set -o xtrace - cd \${build_dir} - if [ -f ./test/percona-server-5.7.properties ]; then - . ./test/percona-server-5.7.properties + if [ "$DOCKER_OS" = "none" ]; then + cd \${build_dir} + if [ -f ./test/percona-server-5.7.properties ]; then + . ./test/percona-server-5.7.properties + fi + sudo bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} + else + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + export build_dir=\$(pwd -P) + set -o xtrace + cd \${build_dir} + if [ -f ./test/percona-server-5.7.properties ]; then + . ./test/percona-server-5.7.properties + fi + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} + " fi - sudo bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} """ } } @@ -67,9 +89,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string(defaultValue: 'https://github.com/percona/percona-server-private.git', description: 'github repository for build', name: 'GIT_REPO') string(defaultValue: 'release-5.7.44-52', description: 'Tag/Branch for percona-server repository', name: 'BRANCH') string(defaultValue: '0', description: 'PerconaFT repository', name: 'PERCONAFT_REPO') @@ -96,7 +122,7 @@ parameters { stages { stage('Create PS source tarball') { agent { - label 'min-bionic-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bionic-x64' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") @@ -117,40 +143,40 @@ parameters { } stash includes: 'uploadPath', name: 'uploadPath' stash includes: 'test/percona-server-5.7.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PS generic source packages') { parallel { stage('Build PS generic source rpm') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-centos-7-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PS generic source deb') { agent { - label 'min-bionic-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bionic-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -159,91 +185,91 @@ parameters { parallel { stage('Centos 7') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-centos-7-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-ol-8-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-ol-9-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Bionic(18.04)') { agent { - label 'min-bionic-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bionic-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-focal-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-jammy-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'min-noble-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-noble-x64' } steps { script { @@ -253,304 +279,304 @@ parameters { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Buster(10)') { agent { - label 'min-buster-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-buster-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bullseye-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bookworm-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 tarball') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-centos-7-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Centos 7 debug tarball') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-centos-7-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Centos 8 tarball') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-ol-8-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:8", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Centos 8 debug tarball') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-ol-8-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:8", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Oracle Linux 9 tarball') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-ol-9-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Oracle Linux 9 debug tarball') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-ol-9-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Bionic tarball') { agent { - label 'min-bionic-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bionic-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Bionic debug tarball') { agent { - label 'min-bionic-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bionic-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Focal tarball') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-focal-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Focal debug tarball') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-focal-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy tarball') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-jammy-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy debug tarball') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-jammy-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Buster tarball') { agent { - label 'min-buster-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-buster-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:buster", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Buster debug tarball') { agent { - label 'min-buster-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-buster-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:buster", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Bullseye tarball') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bullseye-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Bullseye debug tarball') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bullseye-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Bookworm tarball') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bookworm-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Bookworm debug tarball') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bookworm-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Upload packages and tarballs from S3') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-jammy-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Sign packages') { @@ -559,9 +585,15 @@ parameters { signDEB() } } + stage('Push to public repository') { + steps { + // sync packages + sync2PrivateProdAutoBuild(params.CLOUD, "ps-57-eol", COMPONENT) + } + } stage('Build docker container') { agent { - label 'min-buster-x64' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'min-buster-x64' } steps { script { @@ -579,11 +611,23 @@ parameters { PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | awk '{print substr($0, 0, 4)}') PS_MAJOR_MINOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | awk '{print substr($0, 0, 7)}' | sed "s/-//g") + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common - sudo apt-get install -y docker.io - sudo systemctl status docker - sudo apt-get install -y qemu binfmt-support qemu-user-static + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo lscpu | grep -q 'sse4_2' && grep -q 'popcnt' /proc/cpuinfo && echo "Supports x86-64-v2" || echo "Does NOT support x86-64-v2" sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker cd percona-docker/percona-server-5.7 mv /tmp/*.rpm . @@ -595,6 +639,8 @@ parameters { sudo docker tag percona/percona-server:${PS_RELEASE}.${RPM_RELEASE} percona/percona-server:${PS_MAJOR_MINOR_RELEASE} sudo docker images sudo docker save -o percona-server-${PS_RELEASE}-${RPM_RELEASE}.docker.tar percona/percona-server:${PS_RELEASE}.${RPM_RELEASE} percona/percona-server:${PS_RELEASE} percona/percona-server:${PS_MAJOR_RELEASE} percona/percona-server:${PS_MAJOR_MINOR_RELEASE} + sudo addgroup admin + sudo useradd -m -s /bin/bash -g admin -G admin admin sudo chown admin:admin percona-server-${PS_RELEASE}-${RPM_RELEASE}.docker.tar sudo chmod a+r percona-server-${PS_RELEASE}-${RPM_RELEASE}.docker.tar ls -la @@ -609,18 +655,11 @@ parameters { } } } - stage('Push to public repository') { - steps { - // sync packages - //sync2ProdAutoBuild('ps-57', COMPONENT) - sync2PrivateProdAutoBuild("ps-57-eol", COMPONENT) - } - } stage('Push Tarballs to TESTING download area') { steps { script { try { - uploadTarballToDownloadsTesting("ps-gated", "${BRANCH}") + uploadTarballToDownloadsTesting(params.CLOUD, "ps-gated", "${BRANCH}") } catch (err) { echo "Caught: ${err}" @@ -632,11 +671,8 @@ parameters { } post { success { - slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") unstash 'properties' - script { - currentBuild.description = "Built on ${BRANCH}; path to packages: ${COMPONENT}/${AWS_STASH_PATH}" - } deleteDir() } failure { @@ -647,6 +683,9 @@ parameters { sh ''' sudo rm -rf ./* ''' + script { + currentBuild.description = "Built on ${BRANCH}; path to packages: ${COMPONENT}/${AWS_STASH_PATH}" + } deleteDir() } } diff --git a/ps/jenkins/percona-server-for-mysql-5.7.yml b/ps/jenkins/percona-server-for-mysql-5.7.yml index 833a8546c1..4b36362e39 100644 --- a/ps/jenkins/percona-server-for-mysql-5.7.yml +++ b/ps/jenkins/percona-server-for-mysql-5.7.yml @@ -1,5 +1,5 @@ - job: - name: ps5.7-autobuild-RELEASE + name: hetzner-ps5.7-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/percona-server-for-mysql-5.7.groovy diff --git a/ps/jenkins/percona-server-for-mysql-8.0-arm.groovy b/ps/jenkins/percona-server-for-mysql-8.0-arm.groovy index fb686b0b3d..4799f1bc2b 100644 --- a/ps/jenkins/percona-server-for-mysql-8.0-arm.groovy +++ b/ps/jenkins/percona-server-for-mysql-8.0-arm.groovy @@ -1,5 +1,5 @@ /* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -191,9 +191,13 @@ def install_mysql_shell = 'no' pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string(defaultValue: 'https://github.com/percona/percona-server.git', description: 'github repository for build', name: 'GIT_REPO') string(defaultValue: 'release-8.0.37-29', description: 'Tag/Branch for percona-server repository', name: 'BRANCH') string(defaultValue: '1', description: 'RPM version', name: 'RPM_RELEASE') @@ -254,8 +258,8 @@ parameters { } stash includes: 'uploadPath', name: 'uploadPath' stash includes: 'test/percona-server-8.0.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PS generic source packages') { @@ -278,8 +282,8 @@ parameters { } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } */ @@ -300,8 +304,8 @@ parameters { } } - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -324,7 +328,7 @@ parameters { popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("none", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } @@ -344,14 +348,14 @@ parameters { popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("none", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Centos 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -364,7 +368,7 @@ parameters { popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } @@ -386,12 +390,12 @@ parameters { } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -406,13 +410,13 @@ parameters { } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } */ stage('Ubuntu Focal(20.04)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -425,14 +429,14 @@ parameters { popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1 --with_zenfs=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -447,12 +451,12 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -467,12 +471,12 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -485,14 +489,14 @@ parameters { popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1 --with_zenfs=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Bookworm(12)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -507,7 +511,7 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } /* @@ -526,7 +530,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -546,7 +550,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--debug=1 --build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -566,7 +570,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -586,7 +590,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--debug=1 --build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -608,7 +612,7 @@ parameters { } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ZenFS tarball') { @@ -625,7 +629,7 @@ parameters { echo "The step is skipped" } else { buildStage("none", "--build_tarball=1 --with_zenfs=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -647,7 +651,7 @@ parameters { } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) tarball') { @@ -665,7 +669,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -685,7 +689,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--debug=1 --build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -707,7 +711,7 @@ parameters { } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ZenFS tarball') { @@ -724,7 +728,7 @@ parameters { echo "The step is skipped" } else { buildStage("none", "--build_tarball=1 --with_zenfs=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -746,7 +750,7 @@ parameters { } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } */ @@ -761,9 +765,9 @@ parameters { installCli("deb") unstash 'properties' - // uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - // uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + // uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + // uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } diff --git a/ps/jenkins/percona-server-for-mysql-8.0-arm.yml b/ps/jenkins/percona-server-for-mysql-8.0-arm.yml index c20c049292..69ee6dfff3 100644 --- a/ps/jenkins/percona-server-for-mysql-8.0-arm.yml +++ b/ps/jenkins/percona-server-for-mysql-8.0-arm.yml @@ -1,5 +1,5 @@ - job: - name: ps8.0-arm-RELEASE + name: hetzner-ps8.0-arm-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/percona-server-for-mysql-8.0-arm.groovy diff --git a/ps/jenkins/percona-server-for-mysql-8.0-docker.groovy b/ps/jenkins/percona-server-for-mysql-8.0-docker.groovy new file mode 100644 index 0000000000..dc65516575 --- /dev/null +++ b/ps/jenkins/percona-server-for-mysql-8.0-docker.groovy @@ -0,0 +1,406 @@ +/* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +import groovy.transform.Field + +void installCli(String PLATFORM) { + sh """ + if [ \${CLOUD} = "AWS" ]; then + set -o xtrace + if [ -d aws ]; then + rm -rf aws + fi + if [ ${PLATFORM} = "deb" ]; then + sudo apt-get update + sudo apt-get -y install wget curl unzip + elif [ ${PLATFORM} = "rpm" ]; then + export RHVER=\$(rpm --eval %rhel) + if [ \${RHVER} = "7" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* || true + sudo sed -i 's|#\\s*baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true + if [ -e "/etc/yum.repos.d/CentOS-SCLo-scl.repo" ]; then + cat /etc/yum.repos.d/CentOS-SCLo-scl.repo + fi + fi + sudo yum -y install wget curl unzip + fi + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip + unzip awscliv2.zip + sudo ./aws/install || true + fi + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def installDependencies(def nodeName) { + def aptNodes = ['min-bullseye-x64', 'min-bookworm-x64', 'min-focal-x64', 'min-jammy-x64', 'min-noble-x64'] + def yumNodes = ['min-ol-8-x64', 'min-centos-7-x64', 'min-ol-9-x64', 'min-amazon-2-x64'] + try{ + if (aptNodes.contains(nodeName)) { + if(nodeName == "min-bullseye-x64" || nodeName == "min-bookworm-x64"){ + sh ''' + sudo apt-get update + sudo apt-get install -y ansible git wget + ''' + }else if(nodeName == "min-focal-x64" || nodeName == "min-jammy-x64" || nodeName == "min-noble-x64"){ + sh ''' + sudo apt-get update + sudo apt-get install -y software-properties-common + sudo apt-add-repository --yes --update ppa:ansible/ansible + sudo apt-get install -y ansible git wget + ''' + }else { + error "Node Not Listed in APT" + } + } else if (yumNodes.contains(nodeName)) { + + if(nodeName == "min-centos-7-x64" || nodeName == "min-ol-9-x64"){ + sh ''' + sudo yum install -y epel-release + sudo yum -y update + sudo yum install -y ansible git wget tar + ''' + }else if(nodeName == "min-ol-8-x64"){ + sh ''' + sudo yum install -y epel-release + sudo yum -y update + sudo yum install -y ansible-2.9.27 git wget tar + ''' + }else if(nodeName == "min-amazon-2-x64"){ + sh ''' + sudo amazon-linux-extras install epel + sudo yum -y update + sudo yum install -y ansible git wget + ''' + } + else { + error "Node Not Listed in YUM" + } + } else { + echo "Unexpected node name: ${nodeName}" + } + } catch (Exception e) { + slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: Server Provision for Mini Package Testing for ${nodeName} at ${BRANCH} FAILED !!") + } + +} + +def runPlaybook(def nodeName) { + + try { + def playbook = "ps_lts_innovation.yml" + def playbook_path = "package-testing/playbooks/${playbook}" + + sh ''' + set -xe + git clone --depth 1 https://github.com/Percona-QA/package-testing + ''' + sh """ + set -xe + export install_repo="\${install_repo}" + export client_to_test="ps80" + export check_warning="\${check_warnings}" + export install_mysql_shell="\${install_mysql_shell}" + ansible-playbook \ + --connection=local \ + --inventory 127.0.0.1, \ + --limit 127.0.0.1 \ + ${playbook_path} + """ + } catch (Exception e) { + slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: Mini Package Testing for ${nodeName} at ${BRANCH} FAILED !!!") + mini_test_error="True" + } +} + +def minitestNodes = [ "min-bullseye-x64", + "min-bookworm-x64", + "min-centos-7-x64", + "min-ol-8-x64", + "min-focal-x64", + "min-amazon-2-x64", + "min-jammy-x64", + "min-noble-x64", + "min-ol-9-x64" ] + +@Field def mini_test_error = "False" +def AWS_STASH_PATH +def PS8_RELEASE_VERSION +def product_to_test = 'innovation-lts' +def install_repo = 'testing' +def action_to_test = 'install' +def check_warnings = 'yes' +def install_mysql_shell = 'no' + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } +parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string(defaultValue: 'release-8.0.28-19', description: 'Tag/Branch for percona-server repository', name: 'BRANCH') + string(defaultValue: '1', description: 'RPM version', name: 'RPM_RELEASE') + string(defaultValue: '1', description: 'DEB version', name: 'DEB_RELEASE') + choice( + choices: 'NO\nYES', + description: 'Enable fipsmode', + name: 'FIPSMODE') + choice( + choices: 'testing\nexperimental\nrelease', + description: 'Repo component to push packages to', + name: 'COMPONENT') + choice( + choices: '#releases-ci\n#releases', + description: 'Channel for notifications', + name: 'SLACKNOTIFY') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + + stage('Build docker containers') { + agent { + label 'launcher-x64' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + echo "====> Build docker container" + sh ''' + PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') + PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}') + if [ ${PS_MAJOR_RELEASE} != "80" ]; then + MYSQL_SHELL_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 6)}' | sed 's/-//g') + MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 6)}' | sed 's/-//g') + else + MYSQL_SHELL_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 7)}' | sed 's/-//g') + MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 7)}' | sed 's/-//g') + fi + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo lscpu | grep -q 'sse4_2' && grep -q 'popcnt' /proc/cpuinfo && echo "Supports x86-64-v2" || echo "Does NOT support x86-64-v2" + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + rm -rf percona-docker + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-8.0 + sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${PS_RELEASE}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PS_TELEMETRY_VERSION.*/ENV PS_TELEMETRY_VERSION ${PS_RELEASE}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV MYSQL_SHELL_VERSION.*/ENV MYSQL_SHELL_VERSION ${MYSQL_SHELL_RELEASE}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PS_REPO .*/ENV PS_REPO testing/g" Dockerfile + if [ ${PS_MAJOR_RELEASE} != "80" ]; then + if [ ${PS_MAJOR_RELEASE} = "84" ]; then + sed -i "s/percona-release enable ps-80/percona-release enable ps-84-lts/g" Dockerfile + else + sed -i "s/percona-release enable ps-80/percona-release enable ps-8x-innovation/g" Dockerfile + fi + sed -i "s/percona-release enable mysql-shell/PS_REPO=\"testing\";percona-release enable mysql-shell/g" Dockerfile + fi + sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${PS_RELEASE}.${RPM_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PS_TELEMETRY_VERSION.*/ENV PS_TELEMETRY_VERSION ${PS_RELEASE}-${RPM_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PS_REPO .*/ENV PS_REPO testing/g" Dockerfile.aarch64 + if [ ${PS_MAJOR_RELEASE} != "80" ]; then + if [ ${PS_MAJOR_RELEASE} = "84" ]; then + sed -i "s/percona-release enable ps-80/percona-release enable ps-84-lts/g" Dockerfile.aarch64 + else + sed -i "s/percona-release enable ps-80/percona-release enable ps-8x-innovation/g" Dockerfile.aarch64 + fi + sed -i "s/percona-release enable mysql-shell/PS_REPO=\"testing\";percona-release enable mysql-shell/g" Dockerfile.aarch64 + fi + sudo docker build -t perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 --progress plain --platform="linux/amd64" . + sudo docker buildx build --platform linux/arm64 -t perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 --load -f Dockerfile.aarch64 . + cd ../mysql-router + sed -i "s/ENV ROUTE_VERSION.*/ENV ROUTE_VERSION ${PS_RELEASE}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV MYSQL_SHELL_VERSION.*/ENV MYSQL_SHELL_VERSION ${MYSQL_SHELL_RELEASE}-${RPM_RELEASE}/g" Dockerfile + if [ ${PS_MAJOR_RELEASE} != "80" ]; then + if [ ${PS_MAJOR_RELEASE} = "84" ]; then + sed -i "s/percona-release enable ps-80 testing/percona-release enable ps-84-lts testing/g" Dockerfile + else + sed -i "s/percona-release enable ps-80 testing/percona-release enable ps-8x-innovation testing/g" Dockerfile + fi + fi + sudo docker build -t perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 --platform="linux/amd64" . + sudo docker build -t perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-arm64 --platform="linux/arm64" . + sudo docker tag perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE} + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') + PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | awk '{print substr($0, 0, 3)}') + if [ ${PS_MAJOR_RELEASE} != "80" ]; then + MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 6)}' | sed 's/-//g') + else + MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 7)}' | sed 's/-//g') + fi + sudo docker tag perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 perconalab/percona-server:${PS_RELEASE}-amd64 + sudo docker push perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 + sudo docker push perconalab/percona-server:${PS_RELEASE}-amd64 + sudo docker tag perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 perconalab/percona-server:${PS_RELEASE}-arm64 + sudo docker push perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 + sudo docker push perconalab/percona-server:${PS_RELEASE}-arm64 + sudo docker tag perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE} perconalab/percona-mysql-router:${PS_MAJOR_RELEASE} + sudo docker push perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 + sudo docker push perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-arm64 + sudo docker push perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE} + sudo docker push perconalab/percona-mysql-router:${PS_MAJOR_RELEASE} + ''' + } + sh ''' + PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') + sudo docker manifest create perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} \ + perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 \ + perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') + PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | awk '{print substr($0, 0, 3)}') + PS_MAJOR_FULL_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | sed "s/-.*//g") + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker manifest push perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-server:${PS_RELEASE} perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-server:${PS_MAJOR_FULL_RELEASE} perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-server:${PS_MAJOR_RELEASE} perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} + ''' + } + } + } + } + } + stage('Check by Trivy') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + environment { + TRIVY_LOG = "trivy-high-junit.xml" + } + steps { + script { + try { + // 🔹 Install Trivy if not present + sh ''' + if ! command -v trivy &> /dev/null; then + echo "🔄 Installing Trivy..." + sudo apt-get update + sudo apt-get -y install wget apt-transport-https gnupg lsb-release + wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - + echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list + sudo apt-get update + sudo apt-get -y install trivy + else + echo "✅ Trivy is already installed." + fi + ''' + + // 🔹 Define the image tags + def PS_RELEASE = "${BRANCH}".replace('release-', '') + def imageList = [ + "perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64", + "perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64" + ] + + // 🔹 Scan images and store logs + imageList.each { image -> + echo "🔍 Scanning ${image}..." + def result = sh(script: """#!/bin/bash + set -e + sudo trivy image --quiet \ + --format table \ + --timeout 10m0s \ + --ignore-unfixed \ + --exit-code 1 \ + --scanners vuln \ + --severity HIGH,CRITICAL ${image} + echo "TRIVY_EXIT_CODE=\$?" + """, returnStatus: true) + echo "Actual Trivy exit code: ${result}" + + // 🔴 Fail the build if vulnerabilities are found + if (result != 0) { + sh """ + sudo trivy image --quiet \ + --format table \ + --timeout 10m0s \ + --ignore-unfixed \ + --exit-code 0 \ + --scanners vuln \ + --severity HIGH,CRITICAL ${image} | tee -a ${TRIVY_LOG} + """ + error "❌ Trivy detected vulnerabilities in ${image}. See ${TRIVY_LOG} for details." + } else { + echo "✅ No critical vulnerabilities found in ${image}." + } + } + } catch (Exception e) { + error "❌ Trivy scan failed: ${e.message}" + } // try + } // script + } // steps + } // stage + } + post { + success { + script { + if (env.FIPSMODE == 'YES') { + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: PRO build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") + } else { + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") + } + } + deleteDir() + } + failure { + slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: build failed for ${BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + script { + currentBuild.description = "Built on ${BRANCH}" + } + deleteDir() + } + } +} diff --git a/ps/jenkins/percona-server-for-mysql-8.0-docker.yml b/ps/jenkins/percona-server-for-mysql-8.0-docker.yml new file mode 100644 index 0000000000..ee817071b1 --- /dev/null +++ b/ps/jenkins/percona-server-for-mysql-8.0-docker.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-ps8.0-docker-build + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/adivinho/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ps/jenkins/percona-server-for-mysql-8.0-docker.groovy diff --git a/ps/jenkins/percona-server-for-mysql-8.0.groovy b/ps/jenkins/percona-server-for-mysql-8.0.groovy index fe4ee6c4f9..8391cd3fb2 100644 --- a/ps/jenkins/percona-server-for-mysql-8.0.groovy +++ b/ps/jenkins/percona-server-for-mysql-8.0.groovy @@ -1,5 +1,5 @@ /* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,68 +8,93 @@ import groovy.transform.Field void installCli(String PLATFORM) { sh """ - set -o xtrace - if [ -d aws ]; then - rm -rf aws - fi - if [ ${PLATFORM} = "deb" ]; then - sudo apt-get update - sudo apt-get -y install wget curl unzip - elif [ ${PLATFORM} = "rpm" ]; then - export RHVER=\$(rpm --eval %rhel) - if [ \${RHVER} = "7" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* || true - sudo sed -i 's|#\\s*baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true - if [ -e "/etc/yum.repos.d/CentOS-SCLo-scl.repo" ]; then - cat /etc/yum.repos.d/CentOS-SCLo-scl.repo + if [ \${CLOUD} = "AWS" ]; then + set -o xtrace + if [ -d aws ]; then + rm -rf aws + fi + if [ ${PLATFORM} = "deb" ]; then + sudo apt-get update + sudo apt-get -y install wget curl unzip + elif [ ${PLATFORM} = "rpm" ]; then + export RHVER=\$(rpm --eval %rhel) + if [ \${RHVER} = "7" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* || true + sudo sed -i 's|#\\s*baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true + if [ -e "/etc/yum.repos.d/CentOS-SCLo-scl.repo" ]; then + cat /etc/yum.repos.d/CentOS-SCLo-scl.repo + fi fi + sudo yum -y install wget curl unzip fi - sudo yum -y install wget curl unzip - fi - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip - unzip awscliv2.zip - sudo ./aws/install || true + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip + unzip awscliv2.zip + sudo ./aws/install || true + fi """ } void buildStage(String DOCKER_OS, String STAGE_PARAM) { - sh """ - set -o xtrace - mkdir -p test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/build-ps/percona-server-8.0_builder.sh -O ps_builder.sh || curl \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/build-ps/percona-server-8.0_builder.sh -o ps_builder.sh - if [ ${FIPSMODE} = "YES" ]; then - sed -i 's|percona-server-server/usr|percona-server-server-pro/usr|g' ps_builder.sh - sed -i 's|dbg-package=percona-server-dbg|dbg-package=percona-server-pro-dbg|g' ps_builder.sh - fi - grep "percona-server-server" ps_builder.sh - export build_dir=\$(pwd -P) - if [ "$DOCKER_OS" = "none" ]; then - set -o xtrace - cd \${build_dir} - if [ -f ./test/percona-server-8.0.properties ]; then - . ./test/percona-server-8.0.properties - fi - sudo bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 - if [ ${BUILD_TOKUDB_TOKUBACKUP} = "ON" ]; then - bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --build_tokudb_tokubackup=1 --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} - else - bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} - fi - else - docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " - set -o xtrace - cd \${build_dir} - if [ -f ./test/percona-server-8.0.properties ]; then - . ./test/percona-server-8.0.properties - fi - bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 - if [ ${BUILD_TOKUDB_TOKUBACKUP} = \"ON\" ]; then - bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --build_tokudb_tokubackup=1 --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} - else - bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} - fi" - fi - """ + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ + set -o xtrace + mkdir -p test + if [ \${FIPSMODE} = "YES" ]; then + MYSQL_VERSION_MINOR=\$(curl -s -O \$(echo \${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/\${BRANCH}/MYSQL_VERSION && grep MYSQL_VERSION_MINOR MYSQL_VERSION | awk -F= '{print \$2}') + if [ \${MYSQL_VERSION_MINOR} = "0" ]; then + PRO_BRANCH="8.0" + elif [ \${MYSQL_VERSION_MINOR} = "4" ]; then + PRO_BRANCH="8.4" + else + PRO_BRANCH="trunk" + fi + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o ps_builder.sh \ + "https://api.github.com/repos/percona/percona-server-private-build/contents/build-ps/percona-server-8.0_builder.sh?ref=\${PRO_BRANCH}" + sed -i 's|percona-server-server/usr|percona-server-server-pro/usr|g' ps_builder.sh + sed -i 's|dbg-package=percona-server-dbg|dbg-package=percona-server-pro-dbg|g' ps_builder.sh + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/build-ps/percona-server-8.0_builder.sh -O ps_builder.sh || curl \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/build-ps/percona-server-8.0_builder.sh -o ps_builder.sh + fi + grep "percona-server-server" ps_builder.sh + export build_dir=\$(pwd -P) + if [ "$DOCKER_OS" = "none" ]; then + set -o xtrace + cd \${build_dir} + if [ \${FIPSMODE} = "YES" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/percona-server-private-build.git percona-server-private-build + mv -f \${build_dir}/percona-server-private-build/build-ps \${build_dir}/test/. + fi + if [ -f ./test/percona-server-8.0.properties ]; then + . ./test/percona-server-8.0.properties + fi + sudo bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ ${BUILD_TOKUDB_TOKUBACKUP} = "ON" ]; then + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --build_tokudb_tokubackup=1 --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} + else + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} + fi + else + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + if [ \${FIPSMODE} = "YES" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/percona-server-private-build.git percona-server-private-build + mv -f \${build_dir}/percona-server-private-build/build-ps \${build_dir}/test/. + fi + if [ -f ./test/percona-server-8.0.properties ]; then + . ./test/percona-server-8.0.properties + fi + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ ${BUILD_TOKUDB_TOKUBACKUP} = \"ON\" ]; then + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --build_tokudb_tokubackup=1 --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} + else + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} + fi" + fi + """ + } } void cleanUpWS() { @@ -196,9 +221,13 @@ def install_mysql_shell = 'no' pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string(defaultValue: 'https://github.com/percona/percona-server.git', description: 'github repository for build', name: 'GIT_REPO') string(defaultValue: 'release-8.0.28-19', description: 'Tag/Branch for percona-server repository', name: 'BRANCH') string(defaultValue: '1', description: 'RPM version', name: 'RPM_RELEASE') @@ -242,13 +271,19 @@ parameters { stage('Create PS source tarball') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' } steps { slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") cleanUpWS() installCli("deb") - buildStage("none", "--get_sources=1") + script { + if (env.FIPSMODE == 'YES') { + buildStage("none", "--get_sources=1 --enable_fipsmode=1") + } else { + buildStage("none", "--get_sources=1") + } + } sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-server-8.0.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -263,52 +298,52 @@ parameters { } stash includes: 'uploadPath', name: 'uploadPath' stash includes: 'test/percona-server-8.0.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PS generic source packages') { parallel { stage('Build PS generic source rpm') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_src_rpm=1 --enable_fipsmode=1") + buildStage("centos:7", "--build_src_rpm=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_src_rpm=1") + buildStage("centos:7", "--build_src_rpm=1") } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PS generic source deb') { agent { - label 'min-buster-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_source_deb=1 --enable_fipsmode=1") + buildStage("debian:buster", "--build_source_deb=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_source_deb=1") + buildStage("debian:buster", "--build_source_deb=1") } } - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -317,7 +352,7 @@ parameters { parallel { stage('Centos 7') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -327,17 +362,17 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("none", "--build_rpm=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Oracle Linux 8') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -347,17 +382,17 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("none", "--build_rpm=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Centos 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -367,43 +402,43 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Oracle Linux 9') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_rpm=1 --with_zenfs=1 --enable_fipsmode=1") + buildStage("oraclelinux:9", "--build_rpm=1 --with_zenfs=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_rpm=1 --with_zenfs=1") + buildStage("oraclelinux:9", "--build_rpm=1 --with_zenfs=1") } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") @@ -412,32 +447,52 @@ parameters { } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } -/* stage('Amazon Linux 2023') { + stage('Amazon Linux 2023') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { - cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'YES') { - buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + if (env.FIPSMODE == 'NO') { + echo "The step is skipped" } else { - buildStage("amazonlinux:2023", "--build_rpm=1 --with_zenfs=1") + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } + } + } + stage('Amazon Linux 2023 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + if (env.FIPSMODE == 'NO') { + echo "The step is skipped" + } else { + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } } - }*/ + } stage('Ubuntu Focal(20.04)') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -447,57 +502,57 @@ parameters { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("none", "--build_deb=1 --with_zenfs=1") + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1 --with_zenfs=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Ubuntu Jammy(22.04)') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") + buildStage("ubuntu:jammy", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_deb=1 --with_zenfs=1") + buildStage("ubuntu:jammy", "--build_deb=1 --with_zenfs=1") } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'min-noble-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") + buildStage("ubuntu:noble", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_deb=1 --with_zenfs=1") + buildStage("ubuntu:noble", "--build_deb=1 --with_zenfs=1") } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -507,37 +562,37 @@ parameters { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("none", "--build_deb=1 --with_zenfs=1") + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1 --with_zenfs=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Bookworm(12)') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") + buildStage("debian:bookworm", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_deb=1 --with_zenfs=1") + buildStage("debian:bookworm", "--build_deb=1 --with_zenfs=1") } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -547,23 +602,23 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1 --with_zenfs=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") @@ -572,18 +627,18 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") @@ -592,12 +647,12 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -607,23 +662,23 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1 --with_zenfs=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") @@ -632,12 +687,12 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 binary tarball') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -647,17 +702,17 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--build_tarball=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("centos:7", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Centos 7 debug tarball') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -667,17 +722,17 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--debug=1 --build_tarball=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("centos:7", "--debug=1 --build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Oracle Linux 8 binary tarball') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -687,17 +742,17 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--build_tarball=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Oracle Linux 8 debug tarball') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -707,76 +762,76 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--debug=1 --build_tarball=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--debug=1 --build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Oracle Linux 9 tarball') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_tarball=1 --enable_fipsmode=1") + buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_tarball=1") + buildStage("oraclelinux:9", "--build_tarball=1") } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ZenFS tarball') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { echo "The step is skipped" } else { - buildStage("none", "--build_tarball=1 --with_zenfs=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_tarball=1 --with_zenfs=1") + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Oracle Linux 9 debug tarball') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--debug=1 --build_tarball=1 --enable_fipsmode=1") + buildStage("oraclelinux:9", "--debug=1 --build_tarball=1 --enable_fipsmode=1") } else { - buildStage("none", "--debug=1 --build_tarball=1") + buildStage("oraclelinux:9", "--debug=1 --build_tarball=1") } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) tarball') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -786,17 +841,17 @@ parameters { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--build_tarball=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Ubuntu Focal(20.04) debug tarball') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -806,87 +861,87 @@ parameters { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--debug=1 --build_tarball=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--debug=1 --build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Ubuntu Jammy(22.04) tarball') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_tarball=1 --enable_fipsmode=1") + buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_tarball=1") + buildStage("ubuntu:jammy", "--build_tarball=1") } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ZenFS tarball') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { echo "The step is skipped" } else { - buildStage("none", "--build_tarball=1 --with_zenfs=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_tarball=1 --with_zenfs=1") + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Ubuntu Jammy(22.04) debug tarball') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--debug=1 --build_tarball=1 --enable_fipsmode=1") + buildStage("ubuntu:jammy", "--debug=1 --build_tarball=1 --enable_fipsmode=1") } else { - buildStage("none", "--debug=1 --build_tarball=1") + buildStage("ubuntu:jammy", "--debug=1 --build_tarball=1") } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Upload packages and tarballs from S3') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } @@ -905,22 +960,22 @@ parameters { // sync packages if ("${MYSQL_VERSION_MINOR}" == "0") { if (env.FIPSMODE == 'YES') { - sync2PrivateProdAutoBuild("ps-80-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "ps-80-pro", COMPONENT) } else { - sync2ProdAutoBuild("ps-80", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "ps-80", COMPONENT) } } else { if (env.FIPSMODE == 'YES') { if ("${MYSQL_VERSION_MINOR}" == "4") { - sync2PrivateProdAutoBuild("ps-84-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "ps-84-pro", COMPONENT) } else { - sync2PrivateProdAutoBuild("ps-8x-innovation-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "ps-8x-innovation-pro", COMPONENT) } } else { if ("${MYSQL_VERSION_MINOR}" == "4") { - sync2ProdAutoBuild("ps-84-lts", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "ps-84-lts", COMPONENT) } else { - sync2ProdAutoBuild("ps-8x-innovation", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "ps-8x-innovation", COMPONENT) } } } @@ -932,9 +987,9 @@ parameters { script { try { if (env.FIPSMODE == 'YES') { - uploadTarballToDownloadsTesting("ps-gated", "${BRANCH}") + uploadTarballToDownloadsTesting(params.CLOUD, "ps-gated", "${BRANCH}") } else { - uploadTarballToDownloadsTesting("ps", "${BRANCH}") + uploadTarballToDownloadsTesting(params.CLOUD, "ps", "${BRANCH}") } } catch (err) { @@ -946,7 +1001,7 @@ parameters { } stage('Build docker containers') { agent { - label 'min-focal-x64' + label 'launcher-x64' } steps { script { @@ -954,12 +1009,6 @@ parameters { echo "The step is skipped" } else { echo "====> Build docker container" - cleanUpWS() - installCli("deb") - sh ''' - sleep 1200 - ''' - unstash 'properties' sh ''' PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}') @@ -971,17 +1020,25 @@ parameters { MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 7)}' | sed 's/-//g') fi sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common - sudo apt-get install -y docker.io - sudo systemctl status docker - sudo apt-get install -y qemu binfmt-support qemu-user-static + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo lscpu | grep -q 'sse4_2' && grep -q 'popcnt' /proc/cpuinfo && echo "Supports x86-64-v2" || echo "Does NOT support x86-64-v2" sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes git clone https://github.com/percona/percona-docker cd percona-docker/percona-server-8.0 sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${PS_RELEASE}.${RPM_RELEASE}/g" Dockerfile sed -i "s/ENV PS_TELEMETRY_VERSION.*/ENV PS_TELEMETRY_VERSION ${PS_RELEASE}-${RPM_RELEASE}/g" Dockerfile - if [ ${PS_MAJOR_RELEASE} != "80" ]; then - sed -i "s/ENV MYSQL_SHELL_VERSION.*/ENV MYSQL_SHELL_VERSION ${MYSQL_SHELL_RELEASE}-${RPM_RELEASE}/g" Dockerfile - fi + sed -i "s/ENV MYSQL_SHELL_VERSION.*/ENV MYSQL_SHELL_VERSION ${MYSQL_SHELL_RELEASE}-${RPM_RELEASE}/g" Dockerfile sed -i "s/ENV PS_REPO .*/ENV PS_REPO testing/g" Dockerfile if [ ${PS_MAJOR_RELEASE} != "80" ]; then if [ ${PS_MAJOR_RELEASE} = "84" ]; then @@ -1002,8 +1059,8 @@ parameters { fi sed -i "s/percona-release enable mysql-shell/PS_REPO=\"testing\";percona-release enable mysql-shell/g" Dockerfile.aarch64 fi - sudo docker build -t perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 . - sudo docker build -t perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 --platform="linux/arm64" -f Dockerfile.aarch64 . + sudo docker build -t perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 --progress plain --platform="linux/amd64" . + sudo docker buildx build --platform linux/arm64 -t perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 --load -f Dockerfile.aarch64 . cd ../mysql-router sed -i "s/ENV ROUTE_VERSION.*/ENV ROUTE_VERSION ${PS_RELEASE}.${RPM_RELEASE}/g" Dockerfile sed -i "s/ENV MYSQL_SHELL_VERSION.*/ENV MYSQL_SHELL_VERSION ${MYSQL_SHELL_RELEASE}-${RPM_RELEASE}/g" Dockerfile @@ -1014,7 +1071,9 @@ parameters { sed -i "s/percona-release enable ps-80 testing/percona-release enable ps-8x-innovation testing/g" Dockerfile fi fi - sudo docker build -t perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE} . + sudo docker build -t perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 --platform="linux/amd64" . + sudo docker build -t perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-arm64 --platform="linux/arm64" . + sudo docker tag perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE} sudo docker images ''' withCredentials([ @@ -1031,13 +1090,15 @@ parameters { else MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 7)}' | sed 's/-//g') fi - sudo docker tag perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} perconalab/percona-server:${PS_RELEASE} - sudo docker push perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} - sudo docker push perconalab/percona-server:${PS_RELEASE} - sudo docker tag perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-aarch64 perconalab/percona-server:${PS_RELEASE}-aarch64 - sudo docker push perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-aarch64 - sudo docker push perconalab/percona-server:${PS_RELEASE}-aarch64 + sudo docker tag perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 perconalab/percona-server:${PS_RELEASE}-amd64 + sudo docker push perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 + sudo docker push perconalab/percona-server:${PS_RELEASE}-amd64 + sudo docker tag perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 perconalab/percona-server:${PS_RELEASE}-arm64 + sudo docker push perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 + sudo docker push perconalab/percona-server:${PS_RELEASE}-arm64 sudo docker tag perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE} perconalab/percona-mysql-router:${PS_MAJOR_RELEASE} + sudo docker push perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 + sudo docker push perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-arm64 sudo docker push perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE} sudo docker push perconalab/percona-mysql-router:${PS_MAJOR_RELEASE} ''' @@ -1058,21 +1119,25 @@ parameters { )]) { sh ''' PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') + PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | awk '{print substr($0, 0, 3)}') + PS_MAJOR_FULL_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | sed "s/-.*//g") echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin - PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') sudo docker manifest push perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-server:${PS_RELEASE} perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-server:${PS_MAJOR_FULL_RELEASE} perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-server:${PS_MAJOR_RELEASE} perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} ''' } } } } - } + } } post { success { script { if (env.FIPSMODE == 'YES') { - slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: PRO build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: PRO -> build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") } else { slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") } @@ -1148,16 +1213,19 @@ parameters { deleteDir() } failure { - slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: build failed for ${BRANCH} - [${BUILD_URL}]") - script { - currentBuild.description = "Built on ${BRANCH}" - } deleteDir() } always { sh ''' sudo rm -rf ./* ''' + script { + if (env.FIPSMODE == 'YES') { + currentBuild.description = "Pro -> Build on ${BRANCH}" + } else { + currentBuild.description = "Build on ${BRANCH}" + } + } deleteDir() } } diff --git a/ps/jenkins/percona-server-for-mysql-8.0.yml b/ps/jenkins/percona-server-for-mysql-8.0.yml index d334d9a4de..49d5ad3dee 100644 --- a/ps/jenkins/percona-server-for-mysql-8.0.yml +++ b/ps/jenkins/percona-server-for-mysql-8.0.yml @@ -1,5 +1,5 @@ - job: - name: ps8.0-autobuild-RELEASE + name: hetzner-ps8.0-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/percona-server-for-mysql-8.0.groovy diff --git a/ps/jenkins/percona-server-for-mysql-9.0.groovy b/ps/jenkins/percona-server-for-mysql-9.0.groovy index 000b9fcaaf..bd59243f4c 100644 --- a/ps/jenkins/percona-server-for-mysql-9.0.groovy +++ b/ps/jenkins/percona-server-for-mysql-9.0.groovy @@ -1,5 +1,5 @@ /* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -183,9 +183,13 @@ def install_mysql_shell = 'no' pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string(defaultValue: 'https://github.com/percona/percona-server.git', description: 'github repository for build', name: 'GIT_REPO') string(defaultValue: 'release-9.0.1-1', description: 'Tag/Branch for percona-server repository', name: 'BRANCH') string(defaultValue: '1', description: 'RPM version', name: 'RPM_RELEASE') @@ -242,8 +246,8 @@ parameters { } stash includes: 'uploadPath', name: 'uploadPath' stash includes: 'test/percona-server-9.0.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PS generic source packages') { @@ -264,8 +268,8 @@ parameters { buildStage("none", "--build_src_rpm=1") } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PS generic source deb') { @@ -285,8 +289,8 @@ parameters { } } - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -309,7 +313,7 @@ parameters { buildStage("none", "--build_rpm=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } @@ -317,7 +321,7 @@ parameters { } stage('Centos 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -330,7 +334,7 @@ parameters { popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } @@ -352,14 +356,14 @@ parameters { buildStage("none", "--build_rpm=1 --with_zenfs=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -373,7 +377,7 @@ parameters { buildStage("oraclelinux:9", "--build_rpm=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } @@ -393,7 +397,7 @@ parameters { popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("none", "--build_deb=1 --with_zenfs=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -415,8 +419,9 @@ parameters { buildStage("none", "--build_deb=1 --with_zenfs=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } } } stage('Ubuntu Noble(24.04)') { @@ -434,9 +439,7 @@ parameters { } else { buildStage("none", "--build_deb=1 --with_zenfs=1") } - if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) - } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -455,7 +458,7 @@ parameters { popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("none", "--build_deb=1 --with_zenfs=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -477,14 +480,14 @@ parameters { buildStage("none", "--build_deb=1 --with_zenfs=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -497,7 +500,7 @@ parameters { popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1 --with_zenfs=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -505,7 +508,7 @@ parameters { } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -520,12 +523,12 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -539,14 +542,14 @@ parameters { buildStage("ubuntu:noble", "--build_deb=1 --with_zenfs=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -560,7 +563,7 @@ parameters { buildStage("debian:bullseye", "--build_deb=1 --with_zenfs=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -568,7 +571,7 @@ parameters { } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -583,7 +586,7 @@ parameters { } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -603,7 +606,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--build_tarball=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -624,7 +627,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--debug=1 --build_tarball=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -646,7 +649,7 @@ parameters { buildStage("none", "--build_tarball=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -666,7 +669,7 @@ parameters { } else { buildStage("none", "--build_tarball=1 --with_zenfs=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -688,7 +691,7 @@ parameters { buildStage("none", "--debug=1 --build_tarball=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -708,7 +711,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--build_tarball=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -729,7 +732,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--debug=1 --build_tarball=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -751,7 +754,7 @@ parameters { buildStage("none", "--build_tarball=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -771,7 +774,7 @@ parameters { } else { buildStage("none", "--build_tarball=1 --with_zenfs=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -793,7 +796,7 @@ parameters { buildStage("none", "--debug=1 --build_tarball=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -809,12 +812,12 @@ parameters { installCli("deb") unstash 'properties' - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) script { if (env.EXPERIMENTALMODE == 'NO') { - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } } diff --git a/ps/jenkins/percona-server-for-mysql-9.0.yml b/ps/jenkins/percona-server-for-mysql-9.0.yml index 48674ec333..fd7f00b932 100644 --- a/ps/jenkins/percona-server-for-mysql-9.0.yml +++ b/ps/jenkins/percona-server-for-mysql-9.0.yml @@ -1,5 +1,5 @@ - job: - name: ps9.0-RELEASE + name: hetzner-ps9.0-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/percona-server-for-mysql-9.0.groovy diff --git a/ps/jenkins/qpress.groovy b/ps/jenkins/qpress.groovy index 1c2b1b7a18..4af3c2ed36 100644 --- a/ps/jenkins/qpress.groovy +++ b/ps/jenkins/qpress.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -142,9 +142,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona-lab/qpress-packaging.git', description: 'URL for qpress packaging repository', @@ -196,19 +200,19 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build qpress packages') { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "RPM") sh ''' pwd @@ -217,19 +221,19 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "RPM") sh ''' pwd @@ -238,19 +242,19 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "RPM") sh ''' pwd @@ -259,19 +263,19 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "RPM") sh ''' pwd @@ -280,19 +284,40 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Amazon Linux 2023') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "RPM") + sh ''' + pwd + ls -la test/rpm + cp -r test/srpm . + cp -r test/rpm . + ''' + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Amazon Linux 2023 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("amazonlinux:2023", "RPM") sh ''' pwd @@ -301,19 +326,19 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "DEB") sh ''' pwd @@ -321,17 +346,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "DEB") sh ''' pwd @@ -339,17 +364,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye (11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "DEB") sh ''' pwd @@ -357,17 +382,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye (11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "DEB") sh ''' pwd @@ -375,17 +400,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy (22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "DEB") sh ''' pwd @@ -393,17 +418,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy (22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "DEB") sh ''' pwd @@ -411,17 +436,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm (12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "DEB") sh ''' pwd @@ -429,17 +454,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm (12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "DEB") sh ''' pwd @@ -447,17 +472,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble (24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:noble", "DEB") sh ''' pwd @@ -465,17 +490,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble (24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:noble", "DEB") sh ''' pwd @@ -483,22 +508,22 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild('tools', COMPONENT) + sync2ProdAutoBuild(params.CLOUD, 'tools', COMPONENT) } } diff --git a/ps/jenkins/qpress.yml b/ps/jenkins/qpress.yml index 2171795e63..a164ecf24f 100644 --- a/ps/jenkins/qpress.yml +++ b/ps/jenkins/qpress.yml @@ -1,5 +1,5 @@ - job: - name: qpress-RELEASE + name: hetzner-qpress-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/qpress.groovy diff --git a/psmdb/jenkins/get-psmdb-branches-6.0.groovy b/psmdb/jenkins/get-psmdb-branches-6.0.groovy index 365bf6e30a..73ec4be7d1 100644 --- a/psmdb/jenkins/get-psmdb-branches-6.0.groovy +++ b/psmdb/jenkins/get-psmdb-branches-6.0.groovy @@ -1,13 +1,17 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-for-mongodb repository', @@ -22,10 +26,13 @@ pipeline { stages { stage('Get release branches') { steps { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + script { + String S3_STASH = (params.CLOUD == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (params.CLOUD == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { sh """ EC=0 - aws s3 ls s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_60.properties || EC=\$? + aws s3 ls s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_60.properties ${S3_ENDPOINT} || EC=\$? if [ \${EC} = 1 ]; then LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-6.0\\* | tail -1) @@ -38,10 +45,10 @@ pipeline { echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id_60.properties echo "MONGO_TOOLS_TAG=\${MONGO_TOOLS_TAG}" >> branch_commit_id_60.properties - aws s3 cp branch_commit_id_60.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ + aws s3 cp branch_commit_id_60.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ ${S3_ENDPOINT} echo "START_NEW_BUILD=NO" > startBuild else - aws s3 cp s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_60.properties . + aws s3 cp s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_60.properties . ${S3_ENDPOINT} source branch_commit_id_60.properties LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-6.0\\* | tail -1) @@ -59,11 +66,10 @@ pipeline { echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id_60.properties echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id_60.properties echo "MONGO_TOOLS_TAG=\${MONGO_TOOLS_TAG}" >> branch_commit_id_60.properties - aws s3 cp branch_commit_id_60.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ + aws s3 cp branch_commit_id_60.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ ${S3_ENDPOINT} fi """ } - script { START_NEW_BUILD = sh(returnStdout: true, script: "source startBuild; echo \${START_NEW_BUILD}").trim() BRANCH_NAME = sh(returnStdout: true, script: "source branch_commit_id_60.properties; echo \${BRANCH_NAME}").trim() COMMIT_ID = sh(returnStdout: true, script: "source branch_commit_id_60.properties; echo \${COMMIT_ID}").trim() @@ -86,9 +92,8 @@ pipeline { """ } slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") - build job: 'psmdb60-autobuild-RELEASE', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] - build job: 'psmdb60-aarch64-build', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] - build job: 'psmdb-multijob-testing', propagate: false, wait: false, parameters: [string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE)] + build job: 'hetzner-psmdb60-autobuild-RELEASE', parameters: [string(name: 'CLOUD', value: CLOUD), string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] + build job: 'hetzner-psmdb-multijob-testing', propagate: false, wait: false, parameters: [string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE)] } } stage('Build skipped') { diff --git a/psmdb/jenkins/get-psmdb-branches-6.0.yml b/psmdb/jenkins/get-psmdb-branches-6.0.yml index b08d9537da..0f2b467683 100644 --- a/psmdb/jenkins/get-psmdb-branches-6.0.yml +++ b/psmdb/jenkins/get-psmdb-branches-6.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb60-autobuild + name: hetzner-psmdb60-autobuild project-type: pipeline description: | Do not edit this job through the web! @@ -9,7 +9,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/get-psmdb-branches-6.0.groovy diff --git a/psmdb/jenkins/get-psmdb-branches-7.0.groovy b/psmdb/jenkins/get-psmdb-branches-7.0.groovy index 5896399b45..b5345cb733 100644 --- a/psmdb/jenkins/get-psmdb-branches-7.0.groovy +++ b/psmdb/jenkins/get-psmdb-branches-7.0.groovy @@ -1,13 +1,17 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-for-mongodb repository', @@ -22,10 +26,13 @@ pipeline { stages { stage('Get release branches') { steps { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + script { + String S3_STASH = (params.CLOUD == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (params.CLOUD == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { sh """ EC=0 - aws s3 ls s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_70.properties || EC=\$? + aws s3 ls s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_70.properties ${S3_ENDPOINT} || EC=\$? if [ \${EC} = 1 ]; then LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-7.0\\* | tail -1) @@ -38,10 +45,10 @@ pipeline { echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id_70.properties echo "MONGO_TOOLS_TAG=\${MONGO_TOOLS_TAG}" >> branch_commit_id_70.properties - aws s3 cp branch_commit_id_70.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ + aws s3 cp branch_commit_id_70.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ ${S3_ENDPOINT} echo "START_NEW_BUILD=NO" > startBuild else - aws s3 cp s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_70.properties . + aws s3 cp s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_70.properties . ${S3_ENDPOINT} source branch_commit_id_70.properties LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-7.0\\* | tail -1) @@ -59,11 +66,10 @@ pipeline { echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id_70.properties echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id_70.properties echo "MONGO_TOOLS_TAG=\${MONGO_TOOLS_TAG}" >> branch_commit_id_70.properties - aws s3 cp branch_commit_id_70.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ + aws s3 cp branch_commit_id_70.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ ${S3_ENDPOINT} fi """ } - script { START_NEW_BUILD = sh(returnStdout: true, script: "source startBuild; echo \${START_NEW_BUILD}").trim() BRANCH_NAME = sh(returnStdout: true, script: "source branch_commit_id_70.properties; echo \${BRANCH_NAME}").trim() COMMIT_ID = sh(returnStdout: true, script: "source branch_commit_id_70.properties; echo \${COMMIT_ID}").trim() @@ -86,9 +92,8 @@ pipeline { """ } slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") - build job: 'psmdb70-autobuild-RELEASE', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] - build job: 'psmdb70-aarch64-build', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] - build job: 'psmdb-multijob-testing', propagate: false, wait: false, parameters: [string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE)] + build job: 'hetzner-psmdb70-autobuild-RELEASE', parameters: [string(name: 'CLOUD', value: CLOUD), string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] + build job: 'hetzner-psmdb-multijob-testing', propagate: false, wait: false, parameters: [string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE)] } } stage('Build skipped') { diff --git a/psmdb/jenkins/get-psmdb-branches-7.0.yml b/psmdb/jenkins/get-psmdb-branches-7.0.yml index b903634d6a..df9c78f535 100644 --- a/psmdb/jenkins/get-psmdb-branches-7.0.yml +++ b/psmdb/jenkins/get-psmdb-branches-7.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb70-autobuild + name: hetzner-psmdb70-autobuild project-type: pipeline description: | Do not edit this job through the web! @@ -9,7 +9,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/get-psmdb-branches-7.0.groovy diff --git a/psmdb/jenkins/get-psmdb-branches-8.0.groovy b/psmdb/jenkins/get-psmdb-branches-8.0.groovy index 0445905748..c631df0e0f 100644 --- a/psmdb/jenkins/get-psmdb-branches-8.0.groovy +++ b/psmdb/jenkins/get-psmdb-branches-8.0.groovy @@ -1,13 +1,17 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-for-mongodb repository', @@ -22,10 +26,13 @@ pipeline { stages { stage('Get release branches') { steps { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + script { + String S3_STASH = (params.CLOUD == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (params.CLOUD == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { sh """ EC=0 - aws s3 ls s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_80.properties || EC=\$? + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 ls s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_80.properties ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 || EC=\$? if [ \${EC} = 1 ]; then LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-8.0\\* | tail -1) @@ -38,10 +45,10 @@ pipeline { echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id_80.properties echo "MONGO_TOOLS_TAG=\${MONGO_TOOLS_TAG}" >> branch_commit_id_80.properties - aws s3 cp branch_commit_id_80.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp branch_commit_id_80.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 echo "START_NEW_BUILD=NO" > startBuild else - aws s3 cp s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_80.properties . + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_80.properties . ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 source branch_commit_id_80.properties LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-8.0\\* | tail -1) @@ -59,11 +66,10 @@ pipeline { echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id_80.properties echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id_80.properties echo "MONGO_TOOLS_TAG=\${MONGO_TOOLS_TAG}" >> branch_commit_id_80.properties - aws s3 cp branch_commit_id_80.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp branch_commit_id_80.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 fi """ } - script { START_NEW_BUILD = sh(returnStdout: true, script: "source startBuild; echo \${START_NEW_BUILD}").trim() BRANCH_NAME = sh(returnStdout: true, script: "source branch_commit_id_80.properties; echo \${BRANCH_NAME}").trim() COMMIT_ID = sh(returnStdout: true, script: "source branch_commit_id_80.properties; echo \${COMMIT_ID}").trim() @@ -86,9 +92,8 @@ pipeline { """ } slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") - build job: 'psmdb80-autobuild-RELEASE', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] - build job: 'psmdb80-aarch64-build', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] - build job: 'psmdb-multijob-testing', propagate: false, wait: false, parameters: [string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE)] + build job: 'hetzner-psmdb80-autobuild-RELEASE', parameters: [string(name: 'CLOUD', value: CLOUD), string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] + build job: 'hetzner-psmdb-multijob-testing', propagate: false, wait: false, parameters: [string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE)] } } stage('Build skipped') { diff --git a/psmdb/jenkins/get-psmdb-branches-8.0.yml b/psmdb/jenkins/get-psmdb-branches-8.0.yml index 7c1ad0e1fb..e173237f61 100644 --- a/psmdb/jenkins/get-psmdb-branches-8.0.yml +++ b/psmdb/jenkins/get-psmdb-branches-8.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb80-autobuild + name: hetzner-psmdb80-autobuild project-type: pipeline description: | Do not edit this job through the web! @@ -9,7 +9,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/get-psmdb-branches-8.0.groovy diff --git a/psmdb/jenkins/percona-mongodb-mongosh.groovy b/psmdb/jenkins/percona-mongodb-mongosh.groovy index 7b3069404e..67d1832833 100644 --- a/psmdb/jenkins/percona-mongodb-mongosh.groovy +++ b/psmdb/jenkins/percona-mongodb-mongosh.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -30,9 +30,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/mongodb-js/mongosh.git', description: 'URL for percona-mongodb-backup repository', @@ -54,7 +58,7 @@ pipeline { description: 'VERSION value', name: 'VERSION') string( - defaultValue: 'psmdb-70', + defaultValue: 'psmdb-80', description: 'PSMDB repo name', name: 'PSMDB_REPO') choice( @@ -70,6 +74,9 @@ pipeline { } stages { stage('Create MongoDB Shell source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${MONGOSH_GIT_BRANCH} - [${BUILD_URL}]") cleanUpWS() @@ -86,144 +93,194 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build MongoDB Shell RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8(x86_64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_mongosh=1 --build_variant=rpm-x64") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_mongosh=1 --build_variant=rpm-arm64") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9(x86_64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_mongosh=1 --build_variant=rpm-x64") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Amazon Linux 2023') { + stage('Oracle Linux 9(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_mongosh=1 --build_variant=rpm-arm64") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Amazon Linux 2023(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("amazonlinux:2023", "--build_mongosh=1 --build_variant=rpm-x64") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Amazon Linux 2023(aarch64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_mongosh=1 --build_variant=deb-x64") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_mongosh=1 --build_variant=rpm-arm64") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy (22.04)(x86_64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_mongosh=1 --build_variant=deb-x64") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Jammy (22.04)(aarch64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_mongosh=1 --build_variant=deb-arm64") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble (24.04)(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_mongosh=1 --build_variant=deb-x64") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble (24.04)(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_mongosh=1 --build_variant=deb-arm64") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye (11)(x86_64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_mongosh=1 --build_variant=deb-x64") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bookworm(12)') { + stage('Debian Bookworm (12)(x86_64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_mongosh=1 --build_variant=deb-x64") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Oraclelinux 8 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_mongosh=1 --build_variant=linux-x64") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } + stage('Upload packages and tarballs from S3') { + agent { + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' + } + steps { + cleanUpWS() + + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') + } + } + stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PSMDB_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PSMDB_REPO, COMPONENT) } } @@ -232,7 +289,7 @@ pipeline { success { slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${MONGOSH_GIT_BRANCH} - [${BUILD_URL}]") script { - currentBuild.description = "Built on ${MONGOSH_GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" + currentBuild.description = "[${CLOUD}]: Built on ${MONGOSH_GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" } deleteDir() } diff --git a/psmdb/jenkins/percona-mongodb-mongosh.yml b/psmdb/jenkins/percona-mongodb-mongosh.yml index 29b08536db..97a124b76c 100644 --- a/psmdb/jenkins/percona-mongodb-mongosh.yml +++ b/psmdb/jenkins/percona-mongodb-mongosh.yml @@ -1,5 +1,5 @@ - job: - name: mongosh-autobuild-RELEASE + name: hetzner-mongosh-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-mongodb-mongosh.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-3.6.groovy b/psmdb/jenkins/percona-server-for-mongodb-3.6.groovy index ef63e85f23..a47b8eb7ba 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-3.6.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-3.6.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -90,36 +94,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -128,105 +132,105 @@ pipeline { parallel { stage('Centos 7') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Xenial(16.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Stretch(9)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:stretch", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Buster(10)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Centos 7 debug tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--debug=1") - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } } diff --git a/psmdb/jenkins/percona-server-for-mongodb-3.6.yml b/psmdb/jenkins/percona-server-for-mongodb-3.6.yml index 68520d2e73..4d0a89e195 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-3.6.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-3.6.yml @@ -1,5 +1,5 @@ - job: - name: psmdb36-autobuild-RELEASE + name: hetzner-psmdb36-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-3.6.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-4.0.groovy b/psmdb/jenkins/percona-server-for-mongodb-4.0.groovy index 4aa3de0be0..822526d8b7 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-4.0.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-4.0.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -86,36 +90,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -124,92 +128,92 @@ pipeline { parallel { stage('Centos 7') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Buster(10)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Centos 7 debug tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--debug=1") - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } } diff --git a/psmdb/jenkins/percona-server-for-mongodb-4.0.yml b/psmdb/jenkins/percona-server-for-mongodb-4.0.yml index ea731cc559..57251e7b4b 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-4.0.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-4.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb40-autobuild-RELEASE + name: hetzner-psmdb40-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-4.0.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-4.2.groovy b/psmdb/jenkins/percona-server-for-mongodb-4.2.groovy index efce86b7e2..e67429d901 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-4.2.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-4.2.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -86,36 +90,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -124,130 +128,130 @@ pipeline { parallel { stage('Centos 7') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Buster(10)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 tarball(glibc2.17)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Centos 7 debug binary tarball(glibc2.17)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--debug=1") - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) binary tarball(glibc2.35)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Ubuntu Jammy(22.04) debug binary tarball(glibc2.35)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--debug=1") - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } } diff --git a/psmdb/jenkins/percona-server-for-mongodb-4.2.yml b/psmdb/jenkins/percona-server-for-mongodb-4.2.yml index c6d928738d..7d3aa10dd6 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-4.2.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-4.2.yml @@ -1,5 +1,5 @@ - job: - name: psmdb42-autobuild-RELEASE + name: hetzner-psmdb42-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-4.2.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-4.4.groovy b/psmdb/jenkins/percona-server-for-mongodb-4.4.groovy index a0d87bb204..3be21d9b09 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-4.4.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-4.4.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -70,7 +74,7 @@ pipeline { stages { stage('Create PSMDB source tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -89,36 +93,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("debian:buster", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -127,130 +131,130 @@ pipeline { parallel { stage('Centos 7') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Buster(10)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 binary tarball(glibc2.17)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Centos 7 debug binary tarball(glibc2.17)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--debug=1") - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) binary tarball(glibc2.35)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Ubuntu Jammy(22.04) debug binary tarball(glibc2.35)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--debug=1") - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } } diff --git a/psmdb/jenkins/percona-server-for-mongodb-4.4.yml b/psmdb/jenkins/percona-server-for-mongodb-4.4.yml index 9057a22cd7..2177c434a8 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-4.4.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-4.4.yml @@ -1,5 +1,5 @@ - job: - name: psmdb44-autobuild-RELEASE + name: hetzner-psmdb44-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-4.4.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-5.0.groovy b/psmdb/jenkins/percona-server-for-mongodb-5.0.groovy index e584b3f4c3..f12609d1a0 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-5.0.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-5.0.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -70,7 +74,7 @@ pipeline { stages { stage('Create PSMDB source tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -89,36 +93,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -127,132 +131,132 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Oraclelinux 8 binary tarball(glibc2.28)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Ubuntu Focal(20.04) binary tarball(glibc2.31)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Ubuntu Jammy(22.04) binary tarball(glibc2.35)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Ubuntu Noble(24.04) binary tarball(glibc2.39)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Debian Bullseye(11) binary tarball(glibc2.31)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } } diff --git a/psmdb/jenkins/percona-server-for-mongodb-5.0.yml b/psmdb/jenkins/percona-server-for-mongodb-5.0.yml index fb9a3b3ddc..f5977e245a 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-5.0.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-5.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb50-autobuild-RELEASE + name: hetzner-psmdb50-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-5.0.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-6.0.groovy b/psmdb/jenkins/percona-server-for-mongodb-6.0.groovy index 05fa4f368d..ed0a684dae 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-6.0.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-6.0.groovy @@ -1,22 +1,39 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ void buildStage(String DOCKER_OS, String STAGE_PARAM) { - sh """ - set -o xtrace - mkdir test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh - pwd -P - ls -laR - export build_dir=\$(pwd -P) - docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ set -o xtrace - cd \${build_dir} - bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" + ls -laR ./ + rm -rf test/* + mkdir -p test + if [ \${FULL_FEATURED} = "yes" ]; then + PRO_BRANCH="v6.0" + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o psmdb_builder.sh \ + "https://api.github.com/repos/percona/mongo-build-scripts/contents/scripts/psmdb_builder.sh?ref=\${PRO_BRANCH}" + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh + fi + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FULL_FEATURED} = "yes" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/mongo-build-scripts.git percona-packaging + mv -f \${build_dir}/percona-packaging \${build_dir}/test/. + ls -la \${build_dir}/percona-packaging + fi + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" """ + } } void cleanUpWS() { @@ -29,9 +46,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -58,12 +79,36 @@ pipeline { name: 'PSMDB_REPO') choice( choices: 'no\nyes', - description: 'Enable fipsmode', - name: 'FIPSMODE') + description: 'Enable all pro features', + name: 'FULL_FEATURED') choice( choices: 'laboratory\ntesting\nexperimental', description: 'Repo component to push packages to', name: 'COMPONENT') + choice( + name: 'BUILD_DOCKER', + choices: ['true', 'false'], + description: 'Build and push Docker images (default: true)') + choice( + name: 'BUILD_PACKAGES', + choices: ['true', 'false'], + description: 'Build packages (default: true)') + choice( + name: 'TARGET_REPO', + choices: ['PerconaLab','DockerHub'], + description: 'Target repo for docker image, use DockerHub for release only') + choice( + name: 'PSMDB_REPO_TYPE', + choices: ['testing','release','experimental'], + description: 'Packages repo for docker images') + choice( + name: 'DEBUG', + choices: ['no','yes'], + description: 'Additionally build debug image') + choice( + name: 'TESTS', + choices: ['yes','no'], + description: 'Run tests after building') } options { skipDefaultCheckout() @@ -73,10 +118,22 @@ pipeline { } stages { stage('Create PSMDB source tarball') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") cleanUpWS() - buildStage("oraclelinux:8", "--get_sources=1") + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--get_sources=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--get_sources=1") + } + } sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-server-mongodb-60.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -90,314 +147,398 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_src_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_src_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_src_rpm=1") } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_src_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_src_deb=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_src_deb=1") + buildStage("ubuntu:jammy", "--build_src_deb=1") } } - - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PSMDB RPMs/DEBs/Binary tarballs') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_rpm=1") } } - - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_rpm=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--build_rpm=1") + } + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_rpm=1 --full_featured=1") + } else { + buildStage("oraclelinux:9", "--build_rpm=1") + } + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:9", "--build_rpm=1") } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Amazon Linux 2023') { + stage('Amazon Linux 2023(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_rpm=1 --full_featured=1") } else { buildStage("amazonlinux:2023", "--build_rpm=1") } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Amazon Linux 2023(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_rpm=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_deb=1") + buildStage("amazonlinux:2023", "--build_rpm=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_deb=1 --full_featured=1") } else { buildStage("ubuntu:jammy", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_deb=1 --full_featured=1") + } else { + buildStage("ubuntu:jammy", "--build_deb=1") + } + } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_deb=1 --full_featured=1") } else { buildStage("ubuntu:noble", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_deb=1 --full_featured=1") + } else { + buildStage("ubuntu:noble", "--build_deb=1") + } + } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bullseye", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("debian:bullseye", "--build_deb=1 --full_featured=1") } else { buildStage("debian:bullseye", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Centos 8 binary tarball(glibc2.28)') { + stage('Oracle Linux 8 binary tarball(glibc2.28)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_tarball=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Oracle Linux 9 binary tarball(glibc2.34)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_tarball=1 --full_featured=1") } else { buildStage("oraclelinux:9", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } - stage('Ubuntu Focal(20.04) binary tarball(glibc2.31)') { - when { - expression { env.FIPSMODE != 'yes' } - } + stage('Amazon Linux 2023 binary tarball(glibc2.34)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_tarball=1 --full_featured=1") + } else { + buildStage("amazonlinux:2023", "--build_tarball=1") + } + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + } } } stage('Debian Bullseye(11) binary tarball(glibc2.31)') { when { - expression { env.FIPSMODE != 'yes' } + expression { env.FULL_FEATURED != 'yes' } } agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) binary tarball(glibc2.35)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_tarball=1 --full_featured=1") } else { buildStage("ubuntu:jammy", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Ubuntu Noble(24.04) binary tarball(glibc2.39)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:noble", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_tarball=1 --full_featured=1") } else { buildStage("ubuntu:noble", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } } + + stage('Upload packages and tarballs from S3') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') + } + } + stage('Sign packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { signRPM() signDEB() } } stage('Push to public repository') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { // sync packages script { - if (env.FIPSMODE == 'yes') { + if (env.FULL_FEATURED == 'yes') { // Replace by a new procedure when it's ready - sync2PrivateProdAutoBuild(PSMDB_REPO+"-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, PSMDB_REPO+"-pro", COMPONENT) } else { - sync2ProdAutoBuild(PSMDB_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PSMDB_REPO, COMPONENT) } } } } stage('Push Tarballs to TESTING download area') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { script { - if (env.FIPSMODE == 'yes') { + if (env.FULL_FEATURED == 'yes') { try { - uploadTarballToDownloadsTesting("psmdb-gated", "${PSMDB_VERSION}") + uploadTarballToDownloadsTesting(params.CLOUD, "psmdb-gated", "${PSMDB_VERSION}") } catch (err) { echo "Caught: ${err}" @@ -405,7 +546,7 @@ pipeline { } } else { try { - uploadTarballToDownloadsTesting("psmdb", "${PSMDB_VERSION}") + uploadTarballToDownloadsTesting(params.CLOUD, "psmdb", "${PSMDB_VERSION}") } catch (err) { echo "Caught: ${err}" @@ -415,13 +556,288 @@ pipeline { } } } + stage ('Build docker container for aws ecr') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'AWS_ECR' + } + } + steps { + withCredentials([aws(accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: '8468e4e0-5371-4741-a9bb-7c143140acea', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) { + sh """ + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + if [ -f "/usr/bin/yum" ] ; then sudo yum install -y unzip ; else sudo apt-get update && sudo apt-get -y install unzip ; fi + unzip -o awscliv2.zip + sudo ./aws/install + aws ecr-public get-login-password --region us-east-1 | sudo docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-6.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + sudo docker tag percona-server-mongodb public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-amd64 + sudo docker push public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-amd64 + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-debug + sudo docker push public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-debug + fi + """ + } + } + } + stage('Build docker containers for PerconaLab') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FULL_FEATURED == 'yes') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1500 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-6.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-server-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-server-mongodb-arm64 perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + PSMDB_MAJOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f1) + PSMDB_MINOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f2) + PSMDB_PATCH_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f3) + sudo docker manifest create --amend perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + + sudo docker manifest create --amend perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + sudo docker manifest push perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + sudo docker manifest push perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug perconalab/percona-server-mongodb:${PSMDB_VERSION}-debug + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-debug + fi + + ''' + } + } + } + } + } + stage('Build docker containers for DockerHub registry') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'DockerHub' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FULL_FEATURED == 'yes') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1500 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-6.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-server-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-server-mongodb-arm64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + PSMDB_MAJOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f1) + PSMDB_MINOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f2) + PSMDB_PATCH_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f3) + sudo docker manifest create --amend percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + + sudo docker manifest create --amend percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + sudo docker manifest push percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + sudo docker manifest push percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug percona/percona-server-mongodb:${PSMDB_VERSION}-debug + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-debug + fi + + ''' + } + } + } + } + } + stage ('Run testing job') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TESTS', value: 'yes' + } + } + steps { + script { + def psmdb_image = 'percona/percona-server-mongodb:' + params.PSMDB_VERSION + '-amd64' + if ( params.PSMDB_REPO_TYPE == 'testing' ) { + psmdb_image = 'perconalab/percona-server-mongodb:' + params.PSMDB_VERSION + '-amd64' + } + if ( params.PSMDB_REPO_TYPE == 'experimental' ) { + psmdb_image = 'public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-' + params.PSMDB_VERSION + '-amd64' + } + def pbm_branch = sh(returnStdout: true, script: """ + git clone https://github.com/percona/percona-backup-mongodb.git >/dev/null 2>/dev/null + PBM_RELEASE=\$(cd percona-backup-mongodb && git branch -r | grep release | sed 's|origin/||' | sort --version-sort | tail -1) + echo \$PBM_RELEASE + """).trim() + build job: 'hetzner-pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}")] + } + } + } } post { success { slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") script { - currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" + if (env.FULL_FEATURED == 'yes') { + currentBuild.description = "!!! PRO Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" + } else { + currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" + } } deleteDir() } diff --git a/psmdb/jenkins/percona-server-for-mongodb-6.0.yml b/psmdb/jenkins/percona-server-for-mongodb-6.0.yml index 043bffb44e..f0e2742c76 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-6.0.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-6.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb60-autobuild-RELEASE + name: hetzner-psmdb60-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-6.0.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-7.0.groovy b/psmdb/jenkins/percona-server-for-mongodb-7.0.groovy index 7968083597..33440f4cda 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-7.0.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-7.0.groovy @@ -1,22 +1,40 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ void buildStage(String DOCKER_OS, String STAGE_PARAM) { - sh """ - set -o xtrace - mkdir test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh - pwd -P - ls -laR - export build_dir=\$(pwd -P) - docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ set -o xtrace - cd \${build_dir} - bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" - """ + ls -laR ./ + rm -rf test/* + mkdir -p test + if [ \${FULL_FEATURED} = "yes" ]; then + PRO_BRANCH="v7.0" + curl -H "Authorization: token ${TOKEN}" https://api.github.com/user + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o psmdb_builder.sh \ + "https://api.github.com/repos/percona/mongo-build-scripts/contents/scripts/psmdb_builder.sh?ref=\${PRO_BRANCH}" + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh + fi + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FULL_FEATURED} = "yes" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/mongo-build-scripts.git percona-packaging + mv -f \${build_dir}/percona-packaging \${build_dir}/test/. + ls -la \${build_dir}/percona-packaging + fi + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" + """ + } } void cleanUpWS() { @@ -29,9 +47,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -58,12 +80,36 @@ pipeline { name: 'PSMDB_REPO') choice( choices: 'no\nyes', - description: 'Enable fipsmode', - name: 'FIPSMODE') + description: 'Enable all pro features', + name: 'FULL_FEATURED') choice( choices: 'laboratory\ntesting\nexperimental', description: 'Repo component to push packages to', name: 'COMPONENT') + choice( + name: 'BUILD_DOCKER', + choices: ['true', 'false'], + description: 'Build and push Docker images (default: true)') + choice( + name: 'BUILD_PACKAGES', + choices: ['true', 'false'], + description: 'Build packages (default: true)') + choice( + name: 'TARGET_REPO', + choices: ['PerconaLab','DockerHub'], + description: 'Target repo for docker image, use DockerHub for release only') + choice( + name: 'PSMDB_REPO_TYPE', + choices: ['testing','release','experimental'], + description: 'Packages repo for docker images') + choice( + name: 'DEBUG', + choices: ['no','yes'], + description: 'Additionally build debug image') + choice( + name: 'TESTS', + choices: ['yes','no'], + description: 'Run tests after building') } options { skipDefaultCheckout() @@ -73,13 +119,22 @@ pipeline { } stages { stage('Create PSMDB source tarball') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") cleanUpWS() - buildStage("oraclelinux:8", "--get_sources=1") + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--get_sources=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--get_sources=1") + } + } sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-server-mongodb-70.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -93,351 +148,431 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_src_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_src_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_src_rpm=1") } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_src_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_src_deb=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_src_deb=1") + buildStage("ubuntu:jammy", "--build_src_deb=1") } } - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PSMDB RPMs/DEBs/Binary tarballs') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_rpm=1") } } - - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_rpm=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--build_rpm=1") + } + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:9", "--build_rpm=1") } } - - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - /* stage('Amazon Linux 2023') { + stage('Oracle Linux 9(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_rpm=1 --full_featured=1") + } else { + buildStage("oraclelinux:9", "--build_rpm=1") + } + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Amazon Linux 2023(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_rpm=1 --full_featured=1") } else { buildStage("amazonlinux:2023", "--build_rpm=1") } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } - }*/ - stage('Ubuntu Focal(20.04)') { + } + stage('Amazon Linux 2023(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_rpm=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_deb=1") + buildStage("amazonlinux:2023", "--build_rpm=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_deb=1 --full_featured=1") } else { buildStage("ubuntu:jammy", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_deb=1 --full_featured=1") + } else { + buildStage("ubuntu:jammy", "--build_deb=1") + } + } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_deb=1 --full_featured=1") } else { buildStage("ubuntu:noble", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04)(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bullseye", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_deb=1 --full_featured=1") + } else { + buildStage("ubuntu:noble", "--build_deb=1") + } + } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("debian:bullseye", "--build_deb=1 --full_featured=1") } else { buildStage("debian:bullseye", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bookworm(12)') { + stage('Debian Bookworm(12)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("debian:bookworm", "--build_deb=1 --full_featured=1") } else { buildStage("debian:bookworm", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Oracle Linux 8 binary tarball(glibc2.28)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_tarball=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Oracle Linux 9 binary tarball(glibc2.34)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_tarball=1 --full_featured=1") } else { buildStage("oraclelinux:9", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } - stage('Ubuntu Focal(20.04) binary tarball(glibc2.31)') { - when { - expression { env.FIPSMODE != 'yes' } - } + stage('Amazon Linux 2023 binary tarball(glibc2.34)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_tarball=1 --full_featured=1") + } else { + buildStage("amazonlinux:2023", "--build_tarball=1") + } + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + } } } stage('Ubuntu Jammy(22.04) binary tarball(glibc2.35)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_tarball=1 --full_featured=1") } else { buildStage("ubuntu:jammy", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Ubuntu Noble(24.04) binary tarball(glibc2.39)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:noble", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_tarball=1 --full_featured=1") } else { buildStage("ubuntu:noble", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Debian Bullseye(11) binary tarball(glibc2.31)') { when { - expression { env.FIPSMODE != 'yes' } + expression { env.FULL_FEATURED != 'yes' } } agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) binary tarball(glibc2.36)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bookworm", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("debian:bookworm", "--build_tarball=1 --full_featured=1") } else { buildStage("debian:bookworm", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } } + stage('Upload packages and tarballs from S3') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') + } + } + stage('Sign packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { signRPM() signDEB() } } stage('Push to public repository') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { // sync packages script { - if (env.FIPSMODE == 'yes') { - sync2PrivateProdAutoBuild(PSMDB_REPO+"-pro", COMPONENT) + if (env.FULL_FEATURED == 'yes') { + sync2PrivateProdAutoBuild(params.CLOUD, PSMDB_REPO+"-pro", COMPONENT) } else { - sync2ProdAutoBuild(PSMDB_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PSMDB_REPO, COMPONENT) } } } } stage('Push Tarballs to TESTING download area') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { script { - if (env.FIPSMODE == 'yes') { + if (env.FULL_FEATURED == 'yes') { try { - uploadTarballToDownloadsTesting("psmdb-gated", "${PSMDB_VERSION}") + uploadTarballToDownloadsTesting(params.CLOUD, "psmdb-gated", "${PSMDB_VERSION}") } catch (err) { echo "Caught: ${err}" @@ -445,7 +580,7 @@ pipeline { } } else { try { - uploadTarballToDownloadsTesting("psmdb", "${PSMDB_VERSION}") + uploadTarballToDownloadsTesting(params.CLOUD, "psmdb", "${PSMDB_VERSION}") } catch (err) { echo "Caught: ${err}" @@ -455,13 +590,284 @@ pipeline { } } } + stage ('Build docker containers for aws ecr') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'AWS_ECR' + } + } + steps { + withCredentials([aws(accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: '8468e4e0-5371-4741-a9bb-7c143140acea', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) { + sh """ + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + if [ -f "/usr/bin/yum" ] ; then sudo yum install -y unzip ; else sudo apt-get update && sudo apt-get -y install unzip ; fi + unzip -o awscliv2.zip + sudo ./aws/install + aws ecr-public get-login-password --region us-east-1 | sudo docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-7.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + sudo docker tag percona-server-mongodb public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-amd64 + sudo docker push public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-amd64 + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-debug + sudo docker push public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-debug + fi + """ + } + } + } + stage('Build docker containers for PerconaLab') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FULL_FEATURED == 'yes') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1500 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-7.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-server-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-server-mongodb-arm64 perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + PSMDB_MAJOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f1) + PSMDB_MINOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f2) + PSMDB_PATCH_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f3) + sudo docker manifest create --amend perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + + sudo docker manifest create --amend perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + sudo docker manifest push perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + sudo docker manifest push perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug perconalab/percona-server-mongodb:${PSMDB_VERSION}-debug + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-debug + fi + + ''' + } + } + } + } + } + stage('Build docker containers for DockerHub registry') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'DockerHub' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FULL_FEATURED == 'yes') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1500 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-7.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-server-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-server-mongodb-arm64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + PSMDB_MAJOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f1) + PSMDB_MINOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f2) + PSMDB_PATCH_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f3) + sudo docker manifest create --amend percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux + --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux + --arch amd64 + sudo docker manifest inspect percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + + sudo docker manifest create --amend percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + sudo docker manifest push percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + sudo docker manifest push percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug percona/percona-server-mongodb:${PSMDB_VERSION}-debug + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-debug + fi + + ''' + } + } + } + } + } + stage ('Run testing job') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TESTS', value: 'yes' + } + } + steps { + script { + def psmdb_image = 'percona/percona-server-mongodb:' + params.PSMDB_VERSION + '-amd64' + if ( params.PSMDB_REPO_TYPE == 'testing' ) { + psmdb_image = 'perconalab/percona-server-mongodb:' + params.PSMDB_VERSION + '-amd64' + } + if ( params.PSMDB_REPO_TYPE == 'experimental' ) { + psmdb_image = 'public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-' + params.PSMDB_VERSION + '-amd64' + } + def pbm_branch = sh(returnStdout: true, script: """ + git clone https://github.com/percona/percona-backup-mongodb.git >/dev/null 2>/dev/null + PBM_RELEASE=\$(cd percona-backup-mongodb && git branch -r | grep release | sed 's|origin/||' | sort --version-sort | tail -1) + echo \$PBM_RELEASE + """).trim() + build job: 'hetzner-pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}")] + } + } + } } post { success { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") script { - if (env.FIPSMODE == 'yes') { + if (env.FULL_FEATURED == 'yes') { currentBuild.description = "!!! PRO Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" } else { currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" diff --git a/psmdb/jenkins/percona-server-for-mongodb-7.0.yml b/psmdb/jenkins/percona-server-for-mongodb-7.0.yml index bb8bbcadb9..d8e1958974 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-7.0.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-7.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb70-autobuild-RELEASE + name: hetzner-psmdb70-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-7.0.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-8.0.groovy b/psmdb/jenkins/percona-server-for-mongodb-8.0.groovy index 490334a8a1..a0cba0520c 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-8.0.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-8.0.groovy @@ -1,22 +1,37 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ void buildStage(String DOCKER_OS, String STAGE_PARAM) { - sh """ - set -o xtrace - mkdir test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh - pwd -P - ls -laR - export build_dir=\$(pwd -P) - docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ set -o xtrace - cd \${build_dir} - bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" - """ + mkdir test + if [ \${FULL_FEATURED} = "yes" ]; then + PRO_BRANCH="v8.0" + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o psmdb_builder.sh \ + "https://api.github.com/repos/percona/mongo-build-scripts/contents/scripts/psmdb_builder.sh?ref=\${PRO_BRANCH}" + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh + fi + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FULL_FEATURED} = "yes" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/mongo-build-scripts.git percona-packaging + mv -f \${build_dir}/percona-packaging \${build_dir}/test/. + ls -la \${build_dir}/percona-packaging + fi + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" + """ + } } void cleanUpWS() { @@ -29,9 +44,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -58,12 +77,36 @@ pipeline { name: 'PSMDB_REPO') choice( choices: 'no\nyes', - description: 'Enable fipsmode', - name: 'FIPSMODE') + description: 'Enable all pro features', + name: 'FULL_FEATURED') choice( choices: 'laboratory\ntesting\nexperimental', description: 'Repo component to push packages to', name: 'COMPONENT') + choice( + name: 'BUILD_DOCKER', + choices: ['true', 'false'], + description: 'Build and push Docker images (default: true)') + choice( + name: 'BUILD_PACKAGES', + choices: ['true', 'false'], + description: 'Build packages (default: true)') + choice( + name: 'TARGET_REPO', + choices: ['PerconaLab','DockerHub'], + description: 'Target repo for docker image, use DockerHub for release only') + choice( + name: 'PSMDB_REPO_TYPE', + choices: ['testing','release','experimental'], + description: 'Packages repo for docker images') + choice( + name: 'DEBUG', + choices: ['no','yes'], + description: 'Additionally build debug image') + choice( + name: 'TESTS', + choices: ['yes','no'], + description: 'Run tests after building') } options { skipDefaultCheckout() @@ -73,13 +116,22 @@ pipeline { } stages { stage('Create PSMDB source tarball') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") cleanUpWS() - buildStage("oraclelinux:8", "--get_sources=1") + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--get_sources=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--get_sources=1") + } + } sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-server-mongodb-80.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -93,360 +145,400 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_src_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_src_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_src_rpm=1") } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_src_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_src_deb=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_src_deb=1") + buildStage("ubuntu:jammy", "--build_src_deb=1") } } - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PSMDB RPMs/DEBs/Binary tarballs') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_rpm=1") } } - - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_rpm=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--build_rpm=1") + } + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:9", "--build_rpm=1") } } - - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - /* stage('Amazon Linux 2023') { + stage('Oracle Linux 9(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_rpm=1 --full_featured=1") + } else { + buildStage("oraclelinux:9", "--build_rpm=1") + } + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Amazon Linux 2023(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_rpm=1 --full_featured=1") } else { buildStage("amazonlinux:2023", "--build_rpm=1") } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } - }*/ - stage('Ubuntu Focal(20.04)') { + } + stage('Amazon Linux 2023(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_rpm=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_deb=1") + buildStage("amazonlinux:2023", "--build_rpm=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_deb=1 --full_featured=1") } else { buildStage("ubuntu:jammy", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_deb=1 --full_featured=1") + } else { + buildStage("ubuntu:jammy", "--build_deb=1") + } + } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_deb=1 --full_featured=1") } else { buildStage("ubuntu:noble", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } -/* - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04)(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bullseye", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_deb=1 --full_featured=1") } else { - buildStage("debian:bullseye", "--build_deb=1") + buildStage("ubuntu:noble", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } -*/ - stage('Debian Bookworm(12)') { + stage('Debian Bookworm(12)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("debian:bookworm", "--build_deb=1 --full_featured=1") } else { buildStage("debian:bookworm", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Oracle Linux 8 binary tarball(glibc2.28)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_tarball=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Oracle Linux 9 binary tarball(glibc2.34)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_tarball=1 --full_featured=1") } else { buildStage("oraclelinux:9", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } - stage('Ubuntu Focal(20.04) binary tarball(glibc2.31)') { + stage('Amazon Linux 2023 binary tarball(glibc2.34)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_tarball=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_tarball=1") + buildStage("amazonlinux:2023", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Ubuntu Jammy(22.04) binary tarball(glibc2.35)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_tarball=1 --full_featured=1") } else { buildStage("ubuntu:jammy", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Ubuntu Noble(24.04) binary tarball(glibc2.39)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:noble", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_tarball=1 --full_featured=1") } else { buildStage("ubuntu:noble", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } -/* - stage('Ubuntu Jammy(22.04) debug binary tarball(glibc2.35)') { - agent { - label 'docker-64gb' - } - steps { - cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--debug=1 --enable_fipsmode=1") - } else { - buildStage("ubuntu:jammy", "--debug=1") - } - pushArtifactFolder("debug/", AWS_STASH_PATH) - } - } - } -*/ stage('Debian Bookworm(12) binary tarball(glibc2.36)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bookworm", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("debian:bookworm", "--build_tarball=1 --full_featured=1") } else { buildStage("debian:bookworm", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } } + stage('Upload packages and tarballs from S3') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') + } + } + stage('Sign packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { signRPM() signDEB() } } stage('Push to public repository') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { // sync packages script { - if (env.FIPSMODE == 'yes') { - sync2PrivateProdAutoBuild(PSMDB_REPO+"-pro", COMPONENT) + if (env.FULL_FEATURED == 'yes') { + sync2PrivateProdAutoBuild(params.CLOUD, PSMDB_REPO+"-pro", COMPONENT) } else { - sync2ProdAutoBuild(PSMDB_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PSMDB_REPO, COMPONENT) } } } } stage('Push Tarballs to TESTING download area') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { script { - if (env.FIPSMODE == 'yes') { + if (env.FULL_FEATURED == 'yes') { try { - uploadTarballToDownloadsTesting("psmdb-gated", "${PSMDB_VERSION}") + uploadTarballToDownloadsTesting(params.CLOUD, "psmdb-gated", "${PSMDB_VERSION}") } catch (err) { echo "Caught: ${err}" @@ -454,7 +546,7 @@ pipeline { } } else { try { - uploadTarballToDownloadsTesting("psmdb", "${PSMDB_VERSION}") + uploadTarballToDownloadsTesting(params.CLOUD, "psmdb", "${PSMDB_VERSION}") } catch (err) { echo "Caught: ${err}" @@ -464,13 +556,282 @@ pipeline { } } } + stage ('Build docker container for aws ecr') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'AWS_ECR' + } + } + steps { + withCredentials([aws(accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: '8468e4e0-5371-4741-a9bb-7c143140acea', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) { + sh """ + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + if [ -f "/usr/bin/yum" ] ; then sudo yum install -y unzip ; else sudo apt-get update && sudo apt-get -y install unzip ; fi + unzip -o awscliv2.zip + sudo ./aws/install + aws ecr-public get-login-password --region us-east-1 | sudo docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-8.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + sudo docker tag percona-server-mongodb public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-amd64 + sudo docker push public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-amd64 + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-debug + sudo docker push public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-debug + fi + """ + } + } + } + stage('Build docker containers for PerconaLab') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FULL_FEATURED == 'yes') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1500 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-8.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + echo "Building ARM image" + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-server-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-server-mongodb-arm64 perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + PSMDB_MAJOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f1) + PSMDB_MINOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f2) + PSMDB_PATCH_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f3) + sudo docker manifest create --amend perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + + sudo docker manifest create --amend perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + sudo docker manifest push perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + sudo docker manifest push perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug perconalab/percona-server-mongodb:${PSMDB_VERSION}-debug + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-debug + fi + ''' + } + } + } + } + } + stage('Build docker containers for DockerHub registry') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'DockerHub' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FULL_FEATURED == 'yes') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1500 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-8.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-server-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-server-mongodb-arm64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + PSMDB_MAJOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f1) + PSMDB_MINOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f2) + PSMDB_PATCH_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f3) + sudo docker manifest create --amend percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + + sudo docker manifest create --amend percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + sudo docker manifest push percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + sudo docker manifest push percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug percona/percona-server-mongodb:${PSMDB_VERSION}-debug + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-debug + fi + + ''' + } + } + } + } + } + stage ('Run testing job') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TESTS', value: 'yes' + } + } + steps { + script { + def psmdb_image = 'percona/percona-server-mongodb:' + params.PSMDB_VERSION + '-amd64' + if ( params.PSMDB_REPO_TYPE == 'testing' ) { + psmdb_image = 'perconalab/percona-server-mongodb:' + params.PSMDB_VERSION + '-amd64' + } + if ( params.PSMDB_REPO_TYPE == 'experimental' ) { + psmdb_image = 'public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-' + params.PSMDB_VERSION + '-amd64' + } + def pbm_branch = sh(returnStdout: true, script: """ + git clone https://github.com/percona/percona-backup-mongodb.git >/dev/null 2>/dev/null + PBM_RELEASE=\$(cd percona-backup-mongodb && git branch -r | grep release | sed 's|origin/||' | sort --version-sort | tail -1) + echo \$PBM_RELEASE + """).trim() + build job: 'hetzner-pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}")] + } + } + } } post { success { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") script { - if (env.FIPSMODE == 'yes') { + if (env.FULL_FEATURED == 'yes') { currentBuild.description = "!!! PRO Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" } else { currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" diff --git a/psmdb/jenkins/percona-server-for-mongodb-8.0.yml b/psmdb/jenkins/percona-server-for-mongodb-8.0.yml index a72222a31b..60a92cc4d0 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-8.0.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-8.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb80-autobuild-RELEASE + name: hetzner-psmdb80-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-8.0.groovy diff --git a/psmdb/psmdb-docker-arm.groovy b/psmdb/psmdb-docker-arm.groovy index a35ffdae21..a23fd2d20b 100644 --- a/psmdb/psmdb-docker-arm.groovy +++ b/psmdb/psmdb-docker-arm.groovy @@ -5,12 +5,13 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'docker-64gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') choice(name: 'PSMDB_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PSMDB_VERSION', defaultValue: '6.0.2-1', description: 'PSMDB version') choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') @@ -234,7 +235,7 @@ pipeline { PBM_RELEASE=\$(cd percona-backup-mongodb && git branch -r | grep release | sed 's|origin/||' | sort --version-sort | tail -1) echo \$PBM_RELEASE """).trim() - build job: 'pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'instance', value: 'docker-64gb-aarch64'), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}")] + build job: 'hetzner-pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'instance', value: 'docker-aarch64'), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}")] } } } diff --git a/psmdb/psmdb-docker-arm.yml b/psmdb/psmdb-docker-arm.yml index 1c2c82a79d..ae1c4ac69c 100644 --- a/psmdb/psmdb-docker-arm.yml +++ b/psmdb/psmdb-docker-arm.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-docker-arm + name: hetzner-psmdb-docker-arm project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-docker-arm.groovy diff --git a/psmdb/psmdb-docker-cve.groovy b/psmdb/psmdb-docker-cve.groovy index 477ff63785..a51eb7a2a7 100644 --- a/psmdb/psmdb-docker-cve.groovy +++ b/psmdb/psmdb-docker-cve.groovy @@ -5,7 +5,7 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'master' + label 'launcher-x64' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' diff --git a/psmdb/psmdb-docker-cve.yml b/psmdb/psmdb-docker-cve.yml index 003b98c628..501b6e11b3 100644 --- a/psmdb/psmdb-docker-cve.yml +++ b/psmdb/psmdb-docker-cve.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-docker-cve + name: hetzner-psmdb-docker-cve project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-docker-cve.groovy diff --git a/psmdb/psmdb-docker-pro.groovy b/psmdb/psmdb-docker-pro.groovy index 8acfc40e4a..a630e184b4 100644 --- a/psmdb/psmdb-docker-pro.groovy +++ b/psmdb/psmdb-docker-pro.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) @@ -7,16 +7,21 @@ def REPO_CI_PATH pipeline { agent { - label params.ARCH == 'x86_64' ? 'docker' : 'docker-64gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') choice(name: 'PSMDB_REPO', choices: ['release','testing','experimental'], description: 'percona-release repo to take packages from') string(name: 'PSMDB_VERSION', defaultValue: '6.0.18-15', description: 'PSMDB version, for example: 6.0.18-15 or 7.0.14-8') - string(name: 'TICKET_NAME', defaultValue: '', description: 'Target folder for docker image') - choice(name: 'ARCH', choices: ['x86_64','aarch64'], description: 'Docker arch') + choice(name: 'TARGET_REPO', choices: ['DockerHub','REPO.CI'], description: 'Target destination for docker image, use Percona for release only') + string(name: 'TICKET_NAME', defaultValue: '', description: 'Target folder on repo.ci for docker image if TARGET_REPO is repo.ci') + string(name: 'PD_BRANCH', defaultValue: '', description: 'Branch/commit from percona-docker repository to be used for building images') + booleanParam(name: 'UPDATE_MAJ_REL_TAG', defaultValue: false, description: 'Update major release tag (6.0 for 6.0.20) in DockerHub') + booleanParam(name: 'UPDATE_LATEST_TAG', defaultValue: false, description: 'Update \'latest\' tag in DockerHub') + booleanParam(name: 'IGNORE_TRIVY', defaultValue: false, description: 'Push images despite failed trivy check, use with caution') } options { disableConcurrentBuilds() @@ -29,40 +34,77 @@ pipeline { } } } + stage('Prepare environment') { + steps { + script { + if (params.CLOUD == 'Hetzner') { + sh ''' + sudo apt-get update + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common apparmor apparmor-utils + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo mkdir -p /usr/libexec/docker/cli-plugins + LATEST=\$(curl -s https://api.github.com/repos/docker/buildx/releases/latest | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\\1/') + sudo curl -L "https://github.com/docker/buildx/releases/download/v\${LATEST}/buildx-v\${LATEST}.linux-amd64" \\ + -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + ''' + } else { + sh ''' + sudo yum install -y ca-certificates curl gnupg2 git qemu qemu-user-static || true + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + ''' + } + } + } + } stage ('Build image') { steps { withCredentials([usernamePassword(credentialsId: 'PSMDB_PRIVATE_REPO_ACCESS', passwordVariable: 'PASSWORD', usernameVariable: 'USERNAME')]) { sh """ - MAJ_VER=\$(echo ${params.PSMDB_VERSION} | awk -F "." '{print \$1"."\$2}') + MAJ_VER=\$(echo "${params.PSMDB_VERSION}" | awk -F "." '{print \$1"."\$2}') echo \$MAJ_VER - MIN_VER=\$(echo ${params.PSMDB_VERSION} | awk -F "-" '{print \$1}') + MIN_VER=\$(echo "${params.PSMDB_VERSION}" | awk -F "-" '{print \$1}') echo \$MIN_VER git clone https://github.com/percona/percona-docker cd percona-docker/percona-server-mongodb-\$MAJ_VER - sed -E "s|ENV PSMDB_VERSION (.+)|ENV PSMDB_VERSION ${params.PSMDB_VERSION}|" -i Dockerfile - sed -E "s|ENV PSMDB_REPO (.+)|ENV PSMDB_REPO ${params.PSMDB_REPO}|" -i Dockerfile - REPO_VER=\$(echo ${params.PSMDB_VERSION} | awk -F "." '{print \$1\$2}') - OS_VER=\$(grep -P "ENV OS_VER" Dockerfile | cut -d " " -f 3) - echo \$OS_VER - OS_VER_NUM=\$(echo \$OS_VER | sed "s|el||") - echo \$OS_VER_NUM - PKG_URL="http://repo.percona.com/private/${USERNAME}-${PASSWORD}/psmdb-\$REPO_VER-pro/yum/${params.PSMDB_REPO}/\${OS_VER_NUM}/RPMS/${ARCH}/" - echo \$PKG_URL - sed -E "/curl -Lf -o \\/tmp\\/Percona-Server-MongoDB-server.rpm/d" -i Dockerfile - sed -E "/percona-server-mongodb-mongos-\${FULL_PERCONA_VERSION}/d" -i Dockerfile - sed -E "/percona-server-mongodb-tools-\${FULL_PERCONA_VERSION}/d" -i Dockerfile - sed -E "s|rpmkeys --checksig /tmp/Percona-Server-MongoDB-server.rpm|rpmkeys --checksig /tmp/*.rpm|" -i Dockerfile - sed -E "s|rpm -iv /tmp/Percona-Server-MongoDB-server.rpm --nodeps;|dnf -y install /tmp/Percona-Server-MongoDB-mongos-pro.rpm /tmp/Percona-Server-MongoDB-tools.rpm; rpm -iv /tmp/Percona-Server-MongoDB-server-pro.rpm --nodeps;|" -i Dockerfile - sed -E "s|rm -rf /tmp/Percona-Server-MongoDB-server.rpm|rm -rf /tmp/*.rpm|" -i Dockerfile - sed -E "s|server/LICENSE|server-pro/LICENSE|" -i Dockerfile - - mkdir -p packages - curl -Lf -o ./packages/Percona-Server-MongoDB-server-pro.rpm \$PKG_URL/percona-server-mongodb-server-pro-${params.PSMDB_VERSION}.\$OS_VER.${ARCH}.rpm - curl -Lf -o ./packages/Percona-Server-MongoDB-mongos-pro.rpm \$PKG_URL/percona-server-mongodb-mongos-pro-${params.PSMDB_VERSION}.\$OS_VER.${ARCH}.rpm - curl -Lf -o ./packages/Percona-Server-MongoDB-tools.rpm \$PKG_URL/percona-server-mongodb-tools-${params.PSMDB_VERSION}.\$OS_VER.${ARCH}.rpm - sed -E '/^ARG PERCONA_TELEMETRY_DISABLE/a \\\nCOPY packages/*.rpm /tmp/' -i Dockerfile - cat Dockerfile - docker build . -t percona-server-mongodb-pro:${params.PSMDB_VERSION} + if [ -n "${params.PD_BRANCH}" ]; then + git checkout ${params.PD_BRANCH} + fi + for ARCH in x86_64 aarch64; do + FILENAME=\$(if [ \"\$ARCH\" = \"x86_64\" ]; then echo \"Dockerfile\"; else echo \"Dockerfile.aarch64\"; fi) + sed -E "s|ENV PSMDB_VERSION (.+)|ENV PSMDB_VERSION ${params.PSMDB_VERSION}|" -i "\$FILENAME" + sed -E "s|ENV PSMDB_REPO (.+)|ENV PSMDB_REPO ${params.PSMDB_REPO}|" -i "\$FILENAME" + REPO_VER=\$(echo "${params.PSMDB_VERSION}" | awk -F "." '{print \$1\$2}') + OS_VER=\$(grep -P "ENV OS_VER" Dockerfile | cut -d " " -f 3) + echo \$OS_VER + OS_VER_NUM=\$(echo \$OS_VER | sed "s|el||") + echo \$OS_VER_NUM + PKG_URL="http://repo.percona.com/private/${USERNAME}-${PASSWORD}/psmdb-\$REPO_VER-pro/yum/${params.PSMDB_REPO}/\${OS_VER_NUM}/RPMS/\${ARCH}/" + echo \$PKG_URL + sed -E "/curl -Lf -o \\/tmp\\/Percona-Server-MongoDB-server.rpm/d" -i "\$FILENAME" + sed -E "/percona-server-mongodb-mongos-\${FULL_PERCONA_VERSION}/d" -i "\$FILENAME" + sed -E "/percona-server-mongodb-tools-\${FULL_PERCONA_VERSION}/d" -i "\$FILENAME" + sed -E "s|mongosh |mongosh percona-telemetry-agent |" -i "\$FILENAME" + sed -E "s|rpmkeys --checksig /tmp/Percona-Server-MongoDB-server.rpm|rpmkeys --checksig /tmp/*.rpm|" -i "\$FILENAME" + sed -E "s|rpm -iv /tmp/Percona-Server-MongoDB-server.rpm --nodeps;|rpm -iv /tmp/Percona-Server-MongoDB-mongos-pro.rpm /tmp/Percona-Server-MongoDB-tools.rpm; rpm -iv /tmp/Percona-Server-MongoDB-server-pro.rpm --nodeps;|" -i "\$FILENAME" + sed -E "s|rm -rf /tmp/Percona-Server-MongoDB-server.rpm|rm -rf /tmp/*.rpm|" -i "\$FILENAME" + sed -E "s|server/LICENSE|server-pro/LICENSE|" -i "\$FILENAME" + mkdir -p packages + curl -Lf -o ./packages/Percona-Server-MongoDB-server-pro.rpm \$PKG_URL/percona-server-mongodb-server-pro-${params.PSMDB_VERSION}.\$OS_VER.\${ARCH}.rpm + curl -Lf -o ./packages/Percona-Server-MongoDB-mongos-pro.rpm \$PKG_URL/percona-server-mongodb-mongos-pro-${params.PSMDB_VERSION}.\$OS_VER.\${ARCH}.rpm + curl -Lf -o ./packages/Percona-Server-MongoDB-tools.rpm \$PKG_URL/percona-server-mongodb-tools-${params.PSMDB_VERSION}.\$OS_VER.\${ARCH}.rpm + sed -E '/^ARG PERCONA_TELEMETRY_DISABLE/a \\\nCOPY packages/*.rpm /tmp/' -i "\$FILENAME" + cat "\$FILENAME" + if [ "\$ARCH" = "x86_64" ]; then + docker build --no-cache -t percona-server-mongodb-pro-amd -f "\$FILENAME" . + else + docker build --no-cache -t percona-server-mongodb-pro-arm --platform="linux/arm64" -f "\$FILENAME" . + fi + done rm -rf packages """ } @@ -75,22 +117,16 @@ pipeline { try { sh """ TRIVY_VERSION=\$(curl --silent 'https://api.github.com/repos/aquasecurity/trivy/releases/latest' | grep '"tag_name":' | tr -d '"' | sed -E 's/.*v(.+),.*/\\1/') - if [[ ${ARCH} == "x86_64" ]]; then - TRIVY_ARCH="64bit" - elif [[ ${ARCH} == "aarch64" ]]; then - TRIVY_ARCH="ARM64" - fi - wget https://github.com/aquasecurity/trivy/releases/download/v\${TRIVY_VERSION}/trivy_\${TRIVY_VERSION}_Linux-\${TRIVY_ARCH}.tar.gz - sudo tar zxvf trivy_\${TRIVY_VERSION}_Linux-\${TRIVY_ARCH}.tar.gz -C /usr/local/bin/ - rm trivy_\${TRIVY_VERSION}_Linux-\${TRIVY_ARCH}.tar.gz + wget https://github.com/aquasecurity/trivy/releases/download/v\${TRIVY_VERSION}/trivy_\${TRIVY_VERSION}_Linux-64bit.tar.gz + sudo tar zxvf trivy_\${TRIVY_VERSION}_Linux-64bit.tar.gz -C /usr/local/bin/ wget https://raw.githubusercontent.com/aquasecurity/trivy/v\${TRIVY_VERSION}/contrib/junit.tpl curl https://raw.githubusercontent.com/Percona-QA/psmdb-testing/main/docker/trivyignore -o ".trivyignore" - if [ ${params.PSMDB_REPO} = "release" ]; then - /usr/local/bin/trivy -q image --format template --template @junit.tpl -o trivy-hight-junit.xml \ - --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-server-mongodb-pro:${params.PSMDB_VERSION} + if [ "${params.IGNORE_TRIVY}" = "true" ]; then + /usr/local/bin/trivy -q image --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-server-mongodb-pro-arm + /usr/local/bin/trivy -q image --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-server-mongodb-pro-amd else - /usr/local/bin/trivy -q image --format template --template @junit.tpl -o trivy-hight-junit.xml \ - --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-server-mongodb-pro:${params.PSMDB_VERSION} + /usr/local/bin/trivy -q image --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-server-mongodb-pro-arm + /usr/local/bin/trivy -q image --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-server-mongodb-pro-amd fi """ } catch (Exception e) { @@ -101,17 +137,17 @@ pipeline { } } } - post { - always { - junit testResults: "*-junit.xml", keepLongStdio: true, allowEmptyResults: true, skipPublishingChecks: true - } - } } stage ('Push image to repo.ci') { + when { + environment name: 'TARGET_REPO', value: 'REPO.CI' + } steps { sh """ - docker save -o percona-server-mongodb-pro-${params.PSMDB_VERSION}.tar percona-server-mongodb-pro:${params.PSMDB_VERSION} - gzip percona-server-mongodb-pro-${params.PSMDB_VERSION}.tar + docker save -o percona-server-mongodb-pro-${params.PSMDB_VERSION}-amd64.tar percona-server-mongodb-pro-amd + docker save -o percona-server-mongodb-pro-${params.PSMDB_VERSION}-arm64.tar percona-server-mongodb-pro-arm + gzip percona-server-mongodb-pro-${params.PSMDB_VERSION}-amd64.tar + gzip percona-server-mongodb-pro-${params.PSMDB_VERSION}-arm64.tar echo "UPLOAD/experimental/CUSTOM/${TICKET_NAME}/${JOB_NAME}/${BUILD_NUMBER}" > uploadPath """ script { @@ -122,6 +158,79 @@ pipeline { uploadTarball('docker') } } + stage ('Push images to private percona docker registry') { + when { + environment name: 'TARGET_REPO', value: 'DockerHub' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + docker login -u '${USER}' -p '${PASS}' + MAJ_VER=\$(echo ${params.PSMDB_VERSION} | awk -F "." '{print \$1"."\$2}') + MIN_VER=\$(echo ${params.PSMDB_VERSION} | awk -F "-" '{print \$1}') + + docker tag percona-server-mongodb-pro-amd percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-amd64 + docker tag percona-server-mongodb-pro-arm percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-arm64 + docker push percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-amd64 + docker push percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-arm64 + + docker tag percona-server-mongodb-pro-amd percona/percona-server-mongodb-pro:\$MIN_VER-amd64 + docker tag percona-server-mongodb-pro-arm percona/percona-server-mongodb-pro:\$MIN_VER-arm64 + docker push percona/percona-server-mongodb-pro:\$MIN_VER-amd64 + docker push percona/percona-server-mongodb-pro:\$MIN_VER-arm64 + + docker manifest create percona/percona-server-mongodb-pro:${params.PSMDB_VERSION} \ + percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-amd64 \ + percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-arm64 + docker manifest annotate percona/percona-server-mongodb-pro:${params.PSMDB_VERSION} \ + percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-server-mongodb-pro:${params.PSMDB_VERSION} \ + percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-server-mongodb-pro:${params.PSMDB_VERSION} + docker manifest push percona/percona-server-mongodb-pro:${params.PSMDB_VERSION} + + docker manifest create percona/percona-server-mongodb-pro:\$MIN_VER \ + percona/percona-server-mongodb-pro:\$MIN_VER-amd64 \ + percona/percona-server-mongodb-pro:\$MIN_VER-arm64 + docker manifest annotate percona/percona-server-mongodb-pro:\$MIN_VER \ + percona/percona-server-mongodb-pro:\$MIN_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-server-mongodb-pro:\$MIN_VER \ + percona/percona-server-mongodb-pro:\$MIN_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-server-mongodb-pro:\$MIN_VER + docker manifest push percona/percona-server-mongodb-pro:\$MIN_VER + + if [ ${params.UPDATE_MAJ_REL_TAG} = "true" ]; then + docker tag percona-server-mongodb-pro-amd percona/percona-server-mongodb-pro:\$MAJ_VER-amd64 + docker tag percona-server-mongodb-pro-arm percona/percona-server-mongodb-pro:\$MAJ_VER-arm64 + docker push percona/percona-server-mongodb-pro:\$MAJ_VER-amd64 + docker push percona/percona-server-mongodb-pro:\$MAJ_VER-arm64 + + docker manifest create percona/percona-server-mongodb-pro:\$MAJ_VER \ + percona/percona-server-mongodb-pro:\$MAJ_VER-amd64 \ + percona/percona-server-mongodb-pro:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-server-mongodb-pro:\$MAJ_VER \ + percona/percona-server-mongodb-pro:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-server-mongodb-pro:\$MAJ_VER \ + percona/percona-server-mongodb-pro:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-server-mongodb-pro:\$MAJ_VER + docker manifest push percona/percona-server-mongodb-pro:\$MAJ_VER + fi + + if [ "${params.UPDATE_MAJ_REL_TAG}" = "true" ] && [ "${params.UPDATE_LATEST_TAG}" = "true" ]; then + docker manifest create percona/percona-server-mongodb-pro:latest \ + percona/percona-server-mongodb-pro:\$MAJ_VER-amd64 \ + percona/percona-server-mongodb-pro:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-server-mongodb-pro:latest \ + percona/percona-server-mongodb-pro:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-server-mongodb-pro:latest \ + percona/percona-server-mongodb-pro:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-server-mongodb-pro:latest + docker manifest push percona/percona-server-mongodb-pro:latest + fi + """ + } + } + } } post { always { @@ -132,22 +241,13 @@ pipeline { deleteDir() } success { - slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: Building of PSMDB ${PSMDB_VERSION} repo ${PSMDB_REPO} succeed") - script { - currentBuild.description = "Built on ${PSMDB_VERSION}. Path to packages: ${REPO_CI_PATH}" - } + slackNotify("#mongodb_autofeed", "#00FF00", "[${JOB_NAME}]: Building of PSMDB PRO ${PSMDB_VERSION}, repo ${PSMDB_REPO} succeed") } unstable { - slackNotify("#releases-ci", "#F6F930", "[${JOB_NAME}]: Building of PSMDB ${PSMDB_VERSION} repo ${PSMDB_REPO} unstable - [${BUILD_URL}testReport/]") - script { - currentBuild.description = "Built on ${PSMDB_VERSION}. Path to packages: ${REPO_CI_PATH}" - } + slackNotify("#mongodb_autofeed", "#F6F930", "[${JOB_NAME}]: Building of PSMDB PRO ${PSMDB_VERSION}, repo ${PSMDB_REPO} unstable - [${BUILD_URL}testReport/]") } failure { - slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: Building of PSMDB ${PSMDB_VERSION} repo ${PSMDB_REPO} failed - [${BUILD_URL}]") - script { - currentBuild.description = "Built on ${PSMDB_VERSION}. Path to packages: ${REPO_CI_PATH}" - } + slackNotify("#mongodb_autofeed", "#FF0000", "[${JOB_NAME}]: Building of PSMDB PRO ${PSMDB_VERSION}, repo ${PSMDB_REPO} failed - [${BUILD_URL}]") } - } +} } diff --git a/psmdb/psmdb-docker-pro.yml b/psmdb/psmdb-docker-pro.yml index 3378230517..ac4cb1a4f6 100644 --- a/psmdb/psmdb-docker-pro.yml +++ b/psmdb/psmdb-docker-pro.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-docker-pro + name: hetzner-psmdb-docker-pro project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-docker-pro.groovy diff --git a/psmdb/psmdb-docker.groovy b/psmdb/psmdb-docker.groovy index cd545bda8a..00a45a7573 100644 --- a/psmdb/psmdb-docker.groovy +++ b/psmdb/psmdb-docker.groovy @@ -5,12 +5,13 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') choice(name: 'PSMDB_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PSMDB_VERSION', defaultValue: '6.0.2-1', description: 'PSMDB version') choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') @@ -168,7 +169,7 @@ pipeline { PBM_RELEASE=\$(cd percona-backup-mongodb && git branch -r | grep release | sed 's|origin/||' | sort --version-sort | tail -1) echo \$PBM_RELEASE """).trim() - build job: 'pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}")] + build job: 'hetzner-pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}")] } } } diff --git a/psmdb/psmdb-docker.yml b/psmdb/psmdb-docker.yml index ed4f6d22eb..79ae959506 100644 --- a/psmdb/psmdb-docker.yml +++ b/psmdb/psmdb-docker.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-docker + name: hetzner-psmdb-docker project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-docker.groovy diff --git a/psmdb/psmdb-integration.groovy b/psmdb/psmdb-integration.groovy index a3549e276e..d0fe888ace 100644 --- a/psmdb/psmdb-integration.groovy +++ b/psmdb/psmdb-integration.groovy @@ -5,12 +5,14 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') + string(name: 'TEST_VERSION', defaultValue: 'main', description: 'Integration tests version: main, v3') string(name: 'PSMDB_VERSION', defaultValue: 'latest', description: 'PSMDB version') string(name: 'PBM_VERSION', defaultValue: 'latest', description: 'PBM version') string(name: 'PMM_VERSION', defaultValue: 'latest', description: 'PMM2 agent version') @@ -29,7 +31,7 @@ pipeline { -H "Accept: application/vnd.github.v3+json" \ -H "Authorization: token ${GITHUB_API_TOKEN}" \ "https://api.github.com/repos/Percona-Lab/qa-integration/actions/workflows/PMM_PSMDB_PBM.yml/dispatches" \ - -d '{"ref":"main","inputs":{"psmdb_version":"${params.PSMDB_VERSION}","pbm_version":"${params.PBM_VERSION}","pmm_version":"${params.PMM_VERSION}","pmm_repo":"${params.PMM_REPO}","pmm_image":"${params.PMM_IMAGE}"}}' + -d '{"ref":"${params.TEST_VERSION}","inputs":{"psmdb_version":"${params.PSMDB_VERSION}","pbm_version":"${params.PBM_VERSION}","pmm_version":"${params.PMM_VERSION}","pmm_repo":"${params.PMM_REPO}","pmm_image":"${params.PMM_IMAGE}"}}' """ } } diff --git a/psmdb/psmdb-integration.yml b/psmdb/psmdb-integration.yml index dcc0aee5a6..4e61b72568 100644 --- a/psmdb/psmdb-integration.yml +++ b/psmdb/psmdb-integration.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-integration + name: hetzner-psmdb-integration project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-integration.groovy diff --git a/psmdb/psmdb-multijob-testing.groovy b/psmdb/psmdb-multijob-testing.groovy index 98e94bd052..fc5acc2f66 100644 --- a/psmdb/psmdb-multijob-testing.groovy +++ b/psmdb/psmdb-multijob-testing.groovy @@ -5,12 +5,13 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'master' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'master' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') string(name: 'PSMDB_VERSION', defaultValue: '4.4.17', description: 'PSMDB Version') string(name: 'PSMDB_RELEASE', defaultValue: '17', description: 'PSMDB Release') } @@ -32,12 +33,12 @@ pipeline { build job: 'psmdb-parallel', parameters: [ string(name: 'REPO', value: "testing"), string(name: 'PSMDB_VERSION', value: params.PSMDB_VERSION ), string(name: 'ENABLE_TOOLKIT', value: "false"), string(name: 'TESTING_BRANCH', value: "main") ] } } - stage ('Build docker images and check for vulnerabilities') { - steps { + stage ('Build docker images and check for vulnerabilities') { + steps { script { - def version = params.PSMDB_VERSION + '-' + params.PSMDB_RELEASE - build job: 'psmdb-docker', parameters: [string(name: 'PSMDB_REPO', value: "testing"), string(name: 'PSMDB_VERSION', value: version ), string(name: 'TARGET_REPO', value: "PerconaLab") ] - build job: 'psmdb-docker-arm', parameters: [string(name: 'PSMDB_REPO', value: "testing"), string(name: 'PSMDB_VERSION', value: version ), string(name: 'TARGET_REPO', value: "PerconaLab") ] + def version = params.PSMDB_VERSION + '-' + params.PSMDB_RELEASE + build job: 'hetzner-psmdb-docker', parameters: [string(name: 'PSMDB_REPO', value: "testing"), string(name: 'PSMDB_VERSION', value: version ), string(name: 'TARGET_REPO', value: "PerconaLab") ] + build job: 'hetzner-psmdb-docker-arm', parameters: [string(name: 'PSMDB_REPO', value: "testing"), string(name: 'PSMDB_VERSION', value: version ), string(name: 'TARGET_REPO', value: "PerconaLab") ] } } } @@ -45,8 +46,9 @@ pipeline { steps { script { def version = params.PSMDB_VERSION + '-' + params.PSMDB_RELEASE - build job: 'psmdb-integration', parameters: [string(name: 'PSMDB_VERSION', value: version), string(name: 'PBM_VERSION', value: "latest" ), string(name: 'PMM_VERSION', value: "latest"), string(name: 'PMM_REPO', value: "release"), string(name: 'PMM_IMAGE', value: "percona/pmm-server:latest") ] - build job: 'psmdb-integration', parameters: [string(name: 'PSMDB_VERSION', value: version), string(name: 'PBM_VERSION', value: "latest" ), string(name: 'PMM_VERSION', value: "latest"), string(name: 'PMM_REPO', value: "experimental"), string(name: 'PMM_IMAGE', value: "perconalab/pmm-server:dev-latest") ] + build job: 'hetzner-psmdb-integration', parameters: [string(name: 'TEST_VERSION', value: "main"), string(name: 'PSMDB_VERSION', value: version), string(name: 'PBM_VERSION', value: "latest" ), string(name: 'PMM_VERSION', value: "latest"), string(name: 'PMM_REPO', value: "release"), string(name: 'PMM_IMAGE', value: "percona/pmm-server:2") ] + build job: 'hetzner-psmdb-integration', parameters: [string(name: 'TEST_VERSION', value: "v3"), string(name: 'PSMDB_VERSION', value: version), string(name: 'PBM_VERSION', value: "latest" ), string(name: 'PMM_VERSION', value: "latest"), string(name: 'PMM_REPO', value: "release"), string(name: 'PMM_IMAGE', value: "percona/pmm-server:latest") ] + build job: 'hetzner-psmdb-integration', parameters: [string(name: 'TEST_VERSION', value: "v3"), string(name: 'PSMDB_VERSION', value: version), string(name: 'PBM_VERSION', value: "latest" ), string(name: 'PMM_VERSION', value: "latest"), string(name: 'PMM_REPO', value: "experimental"), string(name: 'PMM_IMAGE', value: "perconalab/pmm-server:3-dev-latest") ] } } } diff --git a/psmdb/psmdb-multijob-testing.yml b/psmdb/psmdb-multijob-testing.yml index 8e38ceaf9f..c2c79ee26c 100644 --- a/psmdb/psmdb-multijob-testing.yml +++ b/psmdb/psmdb-multijob-testing.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-multijob-testing + name: hetzner-psmdb-multijob-testing project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-multijob-testing.groovy diff --git a/psmdb/psmdb-regression.groovy b/psmdb/psmdb-regression.groovy index 387fba2469..8e2c72f817 100644 --- a/psmdb/psmdb-regression.groovy +++ b/psmdb/psmdb-regression.groovy @@ -1,11 +1,11 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'master' + label 'launcher-x64' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' @@ -22,7 +22,7 @@ pipeline { string(name: 'parallelexecutors', defaultValue: '1', description: 'Number of parallel executors') string(name: 'testsuites', defaultValue: 'core,unittests,dbtest', description: 'Comma-separated list of testuites') string(name: 'listsuites', defaultValue: '', description: 'URL with list of testuites') - choice(name: 'instance', choices: ['docker-64gb','docker-64gb-aarch64'], description: 'Ec2 instance type for running suites') + choice(name: 'instance', choices: ['docker-x64','docker-aarch64'], description: 'Hetzner instance type for running suites') string(name: 'paralleljobs', defaultValue: '4', description: 'Number of parallel jobs passed to resmoke.py') booleanParam(name: 'unittests',defaultValue: true, description: 'Check if list of suites contains unittests') booleanParam(name: 'integrationtests',defaultValue: false, description: 'Check if list of suites contains integration tests') diff --git a/psmdb/psmdb-regression.yml b/psmdb/psmdb-regression.yml index 16682a50c4..c8098e9e41 100644 --- a/psmdb/psmdb-regression.yml +++ b/psmdb/psmdb-regression.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-regression + name: hetzner-psmdb-regression project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-regression.groovy diff --git a/psmdb/psmdb-site-check.groovy b/psmdb/psmdb-site-check.groovy index 6f27c7dfae..fe5d4ccbcf 100644 --- a/psmdb/psmdb-site-check.groovy +++ b/psmdb/psmdb-site-check.groovy @@ -5,12 +5,16 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice( + name: 'CLOUD', + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build') string( defaultValue: '7.0.2-1', description: 'PSMDB Version for tests', diff --git a/psmdb/psmdb-site-check.yml b/psmdb/psmdb-site-check.yml index e866533389..844ae07295 100644 --- a/psmdb/psmdb-site-check.yml +++ b/psmdb/psmdb-site-check.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-site-check + name: hetzner-psmdb-site-check project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-site-check.groovy diff --git a/pt/jenkins/percona-toolkit.groovy b/pt/jenkins/percona-toolkit.groovy index 1d474c2636..8045b8204a 100644 --- a/pt/jenkins/percona-toolkit.groovy +++ b/pt/jenkins/percona-toolkit.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -28,9 +28,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-toolkit.git', description: 'URL for percona-toolkit repository', @@ -83,36 +87,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build Percona Toolkit generic source packages') { parallel { stage('Build Percona Toolkit generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build Percona Toolkit generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -121,197 +125,197 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Xenial(16.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } @@ -330,7 +334,89 @@ pipeline { sync2ProdAutoBuild(PT_REPO, COMPONENT) } } + stage('Build docker containers') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1200 + ''' + unstash 'uploadPath' + sh ''' + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-toolkit + sed -i "s/ENV PT_VERSION.*/ENV PT_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/pt release/pt ${COMPONENT}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} . + + sudo docker build --no-cache -t perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 --platform="linux/arm64" -f Dockerfile . + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} + sudo docker push perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 + + PT_MAJOR_VERSION=$(echo $VERSION | cut -d'.' -f1) + PT_MINOR_VERSION=$(echo $VERSION | cut -d'.' -f2) + PT_PATCH_VERSION=$(echo $VERSION | cut -d'.' -f3) + sudo docker manifest create perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION}.${PT_PATCH_VERSION} \ + perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} \ + perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 + sudo docker manifest annotate perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION}.${PT_PATCH_VERSION} perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION}.${PT_PATCH_VERSION} perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION}.${PT_PATCH_VERSION} + sudo docker manifest create perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION} \ + perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} \ + perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 + sudo docker manifest annotate perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION} perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION} perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION} + + sudo docker manifest create perconalab/percona-toolkit:${PT_MAJOR_VERSION} \ + perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} \ + perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 + sudo docker manifest annotate perconalab/percona-toolkit:${PT_MAJOR_VERSION} perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-toolkit:${PT_MAJOR_VERSION} perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-toolkit:${PT_MAJOR_VERSION} + + sudo docker manifest push perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION}.${PT_PATCH_VERSION} + sudo docker manifest push perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION} + sudo docker manifest push perconalab/percona-toolkit:${PT_MAJOR_VERSION} + ''' + } + } + } + } } post { success { diff --git a/pt/jenkins/percona-toolkit.yml b/pt/jenkins/percona-toolkit.yml index 11407be6c7..b1dcb17f49 100644 --- a/pt/jenkins/percona-toolkit.yml +++ b/pt/jenkins/percona-toolkit.yml @@ -1,5 +1,5 @@ - job: - name: percona-toolkit-RELEASE + name: hetzner-percona-toolkit-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pt/jenkins/percona-toolkit.groovy diff --git a/pxb/jenkins/pxb-24-arm.groovy b/pxb/jenkins/pxb-24-arm.groovy index 30a83a4704..1d10e8513c 100644 --- a/pxb/jenkins/pxb-24-arm.groovy +++ b/pxb/jenkins/pxb-24-arm.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-xtrabackup.git', description: 'URL for PXB git repository', @@ -61,7 +65,7 @@ pipeline { stages { stage('Create PXB source tarball') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") @@ -83,8 +87,8 @@ pipeline { XB_VERSION_EXTRA = sh(returnStdout: true, script: "grep 'XB_VERSION_EXTRA' ./test/percona-xtrabackup-2.4.properties | cut -d = -f 2 | sed 's/-//g'").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } @@ -92,15 +96,15 @@ pipeline { parallel { stage('Build PXB generic source rpm') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } } //parallel @@ -110,28 +114,28 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } diff --git a/pxb/jenkins/pxb-24-arm.yml b/pxb/jenkins/pxb-24-arm.yml index ef14c24cb8..7e25a4e08d 100644 --- a/pxb/jenkins/pxb-24-arm.yml +++ b/pxb/jenkins/pxb-24-arm.yml @@ -1,5 +1,5 @@ - job: - name: percona-xtrabackup-2.4-arm-RELEASE + name: hetzner-percona-xtrabackup-2.4-arm-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pxb/jenkins/pxb-24-arm.groovy diff --git a/pxb/jenkins/pxb-24.groovy b/pxb/jenkins/pxb-24.groovy index 153050250a..367fa548f8 100644 --- a/pxb/jenkins/pxb-24.groovy +++ b/pxb/jenkins/pxb-24.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -28,9 +28,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-xtrabackup.git', description: 'URL for PXB git repository', @@ -79,36 +83,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PXB generic source packages') { parallel { stage('Build PXB generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PXB generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -117,145 +121,145 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } /* stage('Amazon Linux 2023') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("amazonlinux:2023", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } }*/ stage('Ubuntu Bionic(18.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Buster(10)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('OracleLinux 8 tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } diff --git a/pxb/jenkins/pxb-24.yml b/pxb/jenkins/pxb-24.yml index e4bb139e2f..0f277a75f1 100644 --- a/pxb/jenkins/pxb-24.yml +++ b/pxb/jenkins/pxb-24.yml @@ -1,5 +1,5 @@ - job: - name: percona-xtrabackup-2.4-RELEASE + name: hetzner-percona-xtrabackup-2.4-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pxb/jenkins/pxb-24.groovy diff --git a/pxb/jenkins/pxb-80-arm.groovy b/pxb/jenkins/pxb-80-arm.groovy index a9da8a92d1..689ed997e9 100644 --- a/pxb/jenkins/pxb-80-arm.groovy +++ b/pxb/jenkins/pxb-80-arm.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-xtrabackup.git', description: 'URL for PXB git repository', @@ -61,7 +65,7 @@ pipeline { stages { stage('Create PXB source tarball') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") @@ -83,8 +87,8 @@ pipeline { XB_VERSION_EXTRA = sh(returnStdout: true, script: "grep 'XB_VERSION_EXTRA' ./test/percona-xtrabackup-8.0.properties | cut -d = -f 2 | sed 's/-//g'").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } @@ -92,28 +96,28 @@ pipeline { parallel { stage('Build PXB generic source rpm') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PXB generic source deb') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -123,93 +127,93 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } diff --git a/pxb/jenkins/pxb-80-arm.yml b/pxb/jenkins/pxb-80-arm.yml index 552a406cf5..964bb5f4c1 100644 --- a/pxb/jenkins/pxb-80-arm.yml +++ b/pxb/jenkins/pxb-80-arm.yml @@ -1,5 +1,5 @@ - job: - name: percona-xtrabackup-8.0-arm-RELEASE + name: hetzner-percona-xtrabackup-8.0-arm-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pxb/jenkins/pxb-80-arm.groovy diff --git a/pxb/jenkins/pxb-80-docker.groovy b/pxb/jenkins/pxb-80-docker.groovy new file mode 100644 index 0000000000..fed079b2af --- /dev/null +++ b/pxb/jenkins/pxb-80-docker.groovy @@ -0,0 +1,257 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +import groovy.transform.Field + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def minitestNodes = [ "min-bullseye-x64", + "min-bookworm-x64", + "min-centos-7-x64", + "min-ol-8-x64", + "min-focal-x64", + "min-jammy-x64", + "min-ol-9-x64" ] + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/percona-xtrabackup.git', + description: 'URL for PXB git repository', + name: 'GIT_REPO') + string( + defaultValue: '8.0', + description: 'Tag/Branch for PXB repository', + name: 'BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + choice( + choices: 'testing\nlaboratory\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Build docker containers') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + echo "====> Build docker containers" + sh ''' + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + curl -O https://raw.githubusercontent.com/percona/percona-xtrabackup/${BRANCH}/XB_VERSION + . ./XB_VERSION + curl -O https://raw.githubusercontent.com/percona/percona-server/refs/heads/${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}/MYSQL_VERSION + . ./MYSQL_VERSION + rm -rf percona-docker + git clone https://github.com/percona/percona-docker + if [ \${MYSQL_VERSION_MINOR} = "0" ]; then + cd percona-docker/percona-xtrabackup-8.0 + else + cd percona-docker/percona-xtrabackup-8.x + fi + sed -i "s/ENV XTRABACKUP_VERSION.*/ENV XTRABACKUP_VERSION ${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.1/g" Dockerfile + if [ \${MYSQL_VERSION_MINOR} = "0" ]; then + sed -i "s/pxb-80 testing/pxb-80 ${COMPONENT}/g" Dockerfile + sed -i "s/ps-80;/ps-80 ${COMPONENT};/g" Dockerfile + else + sed -i "s/pxb-84-lts testing/pxb-84-lts ${COMPONENT}/g" Dockerfile + sed -i "s/ps-84-lts release/ps-84-lts ${COMPONENT}/g" Dockerfile + fi + sudo docker build --no-cache -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --platform "linux/amd64" . + sudo docker build --no-cache -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --platform="linux/arm64" . + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + curl -O https://raw.githubusercontent.com/percona/percona-xtrabackup/${BRANCH}/XB_VERSION + . ./XB_VERSION + sudo docker push perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker push perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + + sudo docker manifest create perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} \ + perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 \ + perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + + sudo docker manifest push perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:latest perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + ''' + } + } // scripts + } // steps + } // stage +stage('Check by Trivy') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + environment { + TRIVY_LOG = "trivy-high-junit.xml" + } + steps { + script { + try { + // 🔹 Fetch XtraBackup Version + echo "🔄 Fetching XtraBackup version..." + sh ''' + curl -O https://raw.githubusercontent.com/percona/percona-xtrabackup/${BRANCH}/XB_VERSION + cat XB_VERSION + ''' + + // ✅ Read the version file and parse it manually (CPS-compatible way) + def versionEnv = [:] + def versionContent = readFile('XB_VERSION') + + // 🔹 Manually split lines and process them + versionContent.split("\n").each { line -> + def parts = line.split('=') + if (parts.length == 2) { + versionEnv[parts[0].trim()] = parts[1].trim() + } + } + + // 🔹 Extract version components + def XB_VERSION_MAJOR = versionEnv['XB_VERSION_MAJOR'] + def XB_VERSION_MINOR = versionEnv['XB_VERSION_MINOR'] + def XB_VERSION_PATCH = versionEnv['XB_VERSION_PATCH'] + def XB_VERSION_EXTRA = versionEnv['XB_VERSION_EXTRA'] + + // 🔹 Install Trivy if not already installed + sh ''' + if ! command -v trivy &> /dev/null; then + echo "🔄 Installing Trivy..." + sudo apt-get update + sudo apt-get -y install wget apt-transport-https gnupg lsb-release + wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - + echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list + sudo apt-get update + sudo apt-get -y install trivy + else + echo "✅ Trivy is already installed." + fi + ''' + + // 🔹 Define the image tags + def imageList = [ + "perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64", + "perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64" + ] + + // 🔹 Scan images and store logs + imageList.each { image -> + echo "🔍 Scanning ${image}..." + def result = sh(script: """#!/bin/bash + LANG=C.UTF-8 sudo trivy image --quiet \ + --format table \ + --timeout 10m0s \ + --ignore-unfixed \ + --exit-code 1 \ + --severity HIGH,CRITICAL ${image} | tee -a ${TRIVY_LOG} + """, returnStatus: true) + + if (result != 0) { + error "❌ Trivy detected vulnerabilities in ${image}. See ${TRIVY_LOG} for details." + } else { + echo "✅ No critical vulnerabilities found in ${image}." + } + } + } catch (Exception e) { + error "❌ Trivy scan failed: ${e.message}" + } + } + } + post { + always { + // 🔹 Archive the report + archiveArtifacts artifacts: "${TRIVY_LOG}", allowEmptyArchive: true + } + } +} +/* + stage('Check by trivy') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + steps { + catchError { + sh ''' + curl -O https://raw.githubusercontent.com/percona/percona-xtrabackup/${BRANCH}/XB_VERSION + . ./XB_VERSION + sudo apt-get update + sudo apt-get -y install wget apt-transport-https gnupg lsb-release + wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - + echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list + sudo apt-get update + sudo apt-get -y install trivy + sudo trivy -q image --format table \ + --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 | tee -a trivy-hight-junit.xml + sudo trivy -q image --format table \ + --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 | tee -a trivy-hight-junit.xml + ''' + } + } + } +*/ + } + post { + always { + sh ''' + sudo rm -rf ./* + ''' + script { + currentBuild.description = "Built on ${BRANCH}" + } + deleteDir() + } + } +} diff --git a/pxb/jenkins/pxb-80-docker.yml b/pxb/jenkins/pxb-80-docker.yml new file mode 100644 index 0000000000..5021470d0b --- /dev/null +++ b/pxb/jenkins/pxb-80-docker.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pxb8.0-docker-build + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pxb/jenkins/pxb-80-docker.groovy + diff --git a/pxb/jenkins/pxb-80.groovy b/pxb/jenkins/pxb-80.groovy index afa44d1b7e..cba53a0568 100644 --- a/pxb/jenkins/pxb-80.groovy +++ b/pxb/jenkins/pxb-80.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -6,18 +6,40 @@ library changelog: false, identifier: 'lib@master', retriever: modernSCM([ import groovy.transform.Field void buildStage(String DOCKER_OS, String STAGE_PARAM) { - sh """ - set -o xtrace - mkdir test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/storage/innobase/xtrabackup/utils/percona-xtrabackup-8.0_builder.sh -O percona-xtrabackup-8.0_builder.sh - pwd -P - export build_dir=\$(pwd -P) - docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " - set -o xtrace - cd \${build_dir} - bash -x ./percona-xtrabackup-8.0_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./percona-xtrabackup-8.0_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" - """ + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ + set -o xtrace + mkdir test + if [ \${FIPSMODE} = "YES" ]; then + MYSQL_VERSION_MINOR=\$(curl -s -O \$(echo \${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/\${BRANCH}/MYSQL_VERSION && grep MYSQL_VERSION_MINOR MYSQL_VERSION | awk -F= '{print \$2}') + if [ \${MYSQL_VERSION_MINOR} = "0" ]; then + PRO_BRANCH="8.0" + elif [ \${MYSQL_VERSION_MINOR} = "4" ]; then + PRO_BRANCH="8.4" + else + PRO_BRANCH="trunk" + fi + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o percona-xtrabackup-8.0_builder.sh \ + "https://api.github.com/repos/percona/percona-xtrabackup-private-build/contents/percona-xtrabackup-8.0_builder.sh?ref=\${PRO_BRANCH}" + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/storage/innobase/xtrabackup/utils/percona-xtrabackup-8.0_builder.sh -O percona-xtrabackup-8.0_builder.sh + fi + pwd -P + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./percona-xtrabackup-8.0_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FIPSMODE} = "YES" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/percona-xtrabackup-private-build.git percona-xtrabackup-private-build + mv -f \${build_dir}/percona-xtrabackup-private-build \${build_dir}/test/. + ls \${build_dir}/test + fi + bash -x ./percona-xtrabackup-8.0_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ + } } void cleanUpWS() { @@ -141,9 +163,13 @@ def git_repo = "https://github.com/Percona-QA/package-testing.git" pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-xtrabackup.git', description: 'URL for PXB git repository', @@ -165,7 +191,7 @@ pipeline { description: 'Enable fipsmode', name: 'FIPSMODE') choice( - choices: 'laboratory\ntesting\nexperimental', + choices: 'testing\nlaboratory\nexperimental', description: 'Repo component to push packages to', name: 'COMPONENT') } @@ -179,7 +205,13 @@ pipeline { steps { // slackNotify("", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") cleanUpWS() - buildStage("ubuntu:focal", "--get_sources=1") + script { + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:focal", "--get_sources=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:focal", "--get_sources=1") + } + } sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-xtrabackup-8.0.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -196,8 +228,8 @@ pipeline { XB_VERSION_EXTRA = sh(returnStdout: true, script: "grep 'XB_VERSION_EXTRA' ./test/percona-xtrabackup-8.0.properties | cut -d = -f 2 | sed 's/-//g'").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } @@ -205,11 +237,11 @@ pipeline { parallel { stage('Build PXB generic source rpm') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("centos:7", "--build_src_rpm=1 --enable_fipsmode=1") @@ -218,17 +250,17 @@ pipeline { } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PXB generic source deb') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:focal", "--build_source_deb=1 --enable_fipsmode=1") @@ -237,8 +269,8 @@ pipeline { } } - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -248,7 +280,7 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -256,18 +288,18 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -275,75 +307,94 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Oracle Linux 9') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") } else { buildStage("oraclelinux:9", "--build_rpm=1") } - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") } else { buildStage("oraclelinux:9", "--build_rpm=1") } - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } -/* stage('Amazon Linux 2023') { + stage('Amazon Linux 2023') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { - cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) - if (env.FIPSMODE == 'YES') { - buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + if (env.FIPSMODE == 'NO') { + echo "The step is skipped" } else { - buildStage("amazonlinux:2023", "--build_rpm=1") + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } + } + } + } + stage('Amazon Linux 2023 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + if (env.FIPSMODE == 'NO') { + echo "The step is skipped" + } else { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } } } - }*/ + } stage('Ubuntu Focal(20.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -351,18 +402,18 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -370,94 +421,94 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") } else { buildStage("ubuntu:jammy", "--build_deb=1") } - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") } else { buildStage("ubuntu:jammy", "--build_deb=1") } - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") } else { buildStage("ubuntu:noble", "--build_deb=1") } - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") } else { buildStage("ubuntu:noble", "--build_deb=1") } - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Debian Bullseye(11)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -465,18 +516,18 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -484,54 +535,54 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Bookworm(12)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") } else { buildStage("debian:bookworm", "--build_deb=1") } - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") } else { buildStage("debian:bookworm", "--build_deb=1") } - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Oracle Linux 8 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -539,37 +590,37 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } } stage('Oracle Linux 9 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") } else { buildStage("oraclelinux:9", "--build_tarball=1") } - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } stage('Ubuntu Focal(20.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -577,56 +628,56 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } } stage('Ubuntu Jammy(22.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") } else { buildStage("ubuntu:jammy", "--build_tarball=1") } - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } stage('Ubuntu Noble(24.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_tarball=1 --enable_fipsmode=1") } else { buildStage("ubuntu:noble", "--build_tarball=1") } - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } stage('Debian Bullseye(11) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -634,31 +685,31 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } } stage('Debian Bookworm(12) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_tarball=1 --enable_fipsmode=1") } else { buildStage("debian:bookworm", "--build_tarball=1") } - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } @@ -671,7 +722,23 @@ pipeline { signDEB() } } - + stage('Push Tarballs to TESTING download area') { + steps { + script { + try { + if (env.FIPSMODE == 'YES') { + uploadTarballToDownloadsTesting(params.CLOUD, "pxb-gated", "${BRANCH}") + } else { + uploadTarballToDownloadsTesting(params.CLOUD, "pxb", "${BRANCH}") + } + } + catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } + } + } stage('Push to public repository') { steps { script { @@ -680,28 +747,114 @@ pipeline { // sync packages if ("${MYSQL_VERSION_MINOR}" == "0") { if (env.FIPSMODE == 'YES') { - sync2PrivateProdAutoBuild("pxb-80-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxb-80-pro", COMPONENT) } else { - sync2ProdAutoBuild("pxb-80", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "pxb-80", COMPONENT) } } else { if (env.FIPSMODE == 'YES') { if ("${MYSQL_VERSION_MINOR}" == "4") { - sync2PrivateProdAutoBuild("pxb-84-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxb-84-pro", COMPONENT) } else { - sync2PrivateProdAutoBuild("pxb-8x-innovation-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxb-8x-innovation-pro", COMPONENT) } } else { if ("${MYSQL_VERSION_MINOR}" == "4") { - sync2ProdAutoBuild("pxb-84-lts", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "pxb-84-lts", COMPONENT) } else { - sync2ProdAutoBuild("pxb-8x-innovation", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "pxb-8x-innovation", COMPONENT) } } } } } } + stage('Build docker containers') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1200 + ''' + unstash 'uploadPath' + sh ''' + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + curl -O https://raw.githubusercontent.com/percona/percona-xtrabackup/${BRANCH}/XB_VERSION + . ./XB_VERSION + curl -O https://raw.githubusercontent.com/percona/percona-server/refs/heads/${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}/MYSQL_VERSION + . ./MYSQL_VERSION + rm -rf percona-docker + git clone https://github.com/percona/percona-docker + if [ \${MYSQL_VERSION_MINOR} = "0" ]; then + cd percona-docker/percona-xtrabackup-8.0 + else + cd percona-docker/percona-xtrabackup-8.x + fi + sed -i "s/ENV XTRABACKUP_VERSION.*/ENV XTRABACKUP_VERSION ${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.1/g" Dockerfile + if [ \${MYSQL_VERSION_MINOR} = "0" ]; then + sed -i "s/pxb-80 testing/pxb-80 ${COMPONENT}/g" Dockerfile + else + sed -i "s/pxb-84-lts testing/pxb-84-lts ${COMPONENT}/g" Dockerfile + fi + sudo docker build --no-cache -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --platform "linux/amd64" . + sudo docker build --no-cache -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --platform="linux/arm64" . + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + curl -O https://raw.githubusercontent.com/percona/percona-xtrabackup/${BRANCH}/XB_VERSION + . ./XB_VERSION + sudo docker push perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker push perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + + sudo docker manifest create perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} \ + perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 \ + perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + + sudo docker manifest push perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:latest perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + ''' + } + } // if + } // scripts + } + } } post { success { diff --git a/pxb/jenkins/pxb-80.yml b/pxb/jenkins/pxb-80.yml index 22c13fa84c..f640673947 100644 --- a/pxb/jenkins/pxb-80.yml +++ b/pxb/jenkins/pxb-80.yml @@ -1,5 +1,5 @@ - job: - name: percona-xtrabackup-8.0-RELEASE + name: hetzner-percona-xtrabackup-8.0-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pxb/jenkins/pxb-80.groovy diff --git a/pxc/jenkins/percona-xtradb-cluster-5.7.groovy b/pxc/jenkins/percona-xtradb-cluster-5.7.groovy index 88c5e2e950..fa82fb7531 100644 --- a/pxc/jenkins/percona-xtradb-cluster-5.7.groovy +++ b/pxc/jenkins/percona-xtradb-cluster-5.7.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -49,9 +49,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-xtradb-cluster-private.git', description: 'URL for percona-xtradb-cluster repository', @@ -115,15 +119,15 @@ pipeline { } stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PXC generic source packages') { parallel { stage('Build PXC generic source rpm') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -132,13 +136,13 @@ pipeline { buildStage("centos:7", "--build_src_rpm=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PXC generic source deb') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -147,8 +151,8 @@ pipeline { buildStage("ubuntu:xenial", "--build_source_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -157,7 +161,7 @@ pipeline { parallel { stage('Centos 7') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -166,12 +170,12 @@ pipeline { buildStage("centos:7", "--build_rpm=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -180,12 +184,12 @@ pipeline { buildStage("centos:8", "--build_rpm=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -194,12 +198,12 @@ pipeline { buildStage("oraclelinux:9", "--build_rpm=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Bionic(18.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -208,12 +212,12 @@ pipeline { buildStage("ubuntu:bionic", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -222,12 +226,12 @@ pipeline { buildStage("ubuntu:focal", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -236,12 +240,12 @@ pipeline { buildStage("ubuntu:jammy", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -250,12 +254,12 @@ pipeline { buildStage("ubuntu:noble", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Buster(10)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -264,12 +268,12 @@ pipeline { buildStage("debian:buster", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -278,12 +282,12 @@ pipeline { buildStage("debian:bullseye", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -292,12 +296,12 @@ pipeline { buildStage("debian:bookworm", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -306,13 +310,13 @@ pipeline { buildStage("centos:7", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } /* stage('Centos 7 debug tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -321,13 +325,13 @@ pipeline { buildStage("centos:7", "--build_tarball=1 --debug=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } */ stage('Centos 8 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -336,12 +340,12 @@ pipeline { buildStage("centos:8", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Centos 9 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -350,12 +354,12 @@ pipeline { buildStage("oraclelinux:9", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Ubuntu Bionic (18.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -364,12 +368,12 @@ pipeline { buildStage("ubuntu:bionic", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -378,12 +382,12 @@ pipeline { buildStage("ubuntu:focal", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -392,12 +396,12 @@ pipeline { buildStage("ubuntu:jammy", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Debian Buster (10) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -406,12 +410,12 @@ pipeline { buildStage("debian:buster", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Debian Bullseye (11) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -420,12 +424,12 @@ pipeline { buildStage("debian:bullseye", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Debian Bookworm (12) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() @@ -434,7 +438,7 @@ pipeline { buildStage("debian:bookworm", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } } @@ -448,9 +452,9 @@ pipeline { installCli("deb") unstash 'pxc-57.properties' - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } stage('Sign packages') { diff --git a/pxc/jenkins/percona-xtradb-cluster-5.7.yml b/pxc/jenkins/percona-xtradb-cluster-5.7.yml index 0ef9231615..e2a8c387c0 100644 --- a/pxc/jenkins/percona-xtradb-cluster-5.7.yml +++ b/pxc/jenkins/percona-xtradb-cluster-5.7.yml @@ -1,5 +1,5 @@ - job: - name: pxc57-autobuild-RELEASE + name: hetzner-pxc57-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pxc/jenkins/percona-xtradb-cluster-5.7.groovy diff --git a/pxc/jenkins/percona-xtradb-cluster-8.0-docker.groovy b/pxc/jenkins/percona-xtradb-cluster-8.0-docker.groovy new file mode 100644 index 0000000000..a232848e51 --- /dev/null +++ b/pxc/jenkins/percona-xtradb-cluster-8.0-docker.groovy @@ -0,0 +1,309 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/percona-xtradb-cluster.git', + description: 'URL for percona-xtradb-cluster repository', + name: 'GIT_REPO') + string( + defaultValue: '8.0', + description: 'Tag/Branch for percona-xtradb-cluster repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + choice( + choices: '#releases-ci\n#releases', + description: 'Channel for notifications', + name: 'SLACKNOTIFY') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Build docker containers') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + steps { + script { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + PXC_RELEASE=$(echo ${GIT_BRANCH} | sed 's/release-//g') + PXC_MAJOR_RELEASE=$(echo ${GIT_BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}') + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION + . ./MYSQL_VERSION + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-xtradb-cluster-${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR} + sed -i "s/ENV PXC_VERSION.*/ENV PXC_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PXC_TELEMETRY_VERSION.*/ENV PXC_TELEMETRY_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PXC_REPO .*/ENV PXC_REPO testing/g" Dockerfile + if [ ${PXC_MAJOR_RELEASE} != "80" ]; then + if [ ${PXC_MAJOR_RELEASE} != "84" ]; then + sed -i "s/pxc-80/pxc-8x-innovation/g" Dockerfile + fi + fi + sudo docker build --no-cache --platform "linux/amd64" -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 . + sudo docker build --no-cache --platform "linux/amd64" --build-arg DEBUG=1 -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-amd64 . + + sed -i "s/ENV PXC_VERSION.*/ENV PXC_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PXC_TELEMETRY_VERSION.*/ENV PXC_TELEMETRY_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}-${RPM_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PXC_REPO .*/ENV PXC_REPO testing/g" Dockerfile.aarch64 + if [ ${PXC_MAJOR_RELEASE} != "80" ]; then + if [ ${PXC_MAJOR_RELEASE} != "84" ]; then + sed -i "s/pxc-80/pxc-8x-innovation/g" Dockerfile.aarch64 + fi + fi + sudo docker build --no-cache -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --platform="linux/arm64" -f Dockerfile.aarch64 . + sudo docker build --no-cache --build-arg DEBUG=1 -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-arm64 --platform="linux/arm64" -f Dockerfile.aarch64 . + + if [ ${PXC_MAJOR_RELEASE} != "84" ]; then + cd ../percona-xtradb-cluster-8.0-backup + else + cd ../percona-xtradb-cluster-8.4-backup + fi + sed -i "s/ENV PXC_VERSION.*/ENV PXC_VERSION=${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PXC_REPO.*/ENV PXC_REPO=testing/g" Dockerfile + sed -i "s:yum/release:yum/testing:g" Dockerfile + if [ ${PXC_MAJOR_RELEASE} != "80" ]; then + #sed -i "s/ENV PXB_VERSION.*/ENV PXB_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PXB_VERSION.*/ENV PXB_VERSION 8.4.0-2.1/g" Dockerfile + sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile + if [ ${PXC_MAJOR_RELEASE} != "84" ]; then + sed -i "s/tools/pxb-8x-innovation/g" Dockerfile + sed -i "s/ps-80/ps-8x-innovation/g" Dockerfile + sed -i "s/pxc-80/pxc-8x-innovation/g" Dockerfile + else + sed -i "s/tools/pxb-84-lts/g" Dockerfile + sed -i "s/ps-80/ps-84-lts/g" Dockerfile + sed -i "s/pxc-80/pxc-84-lts/g" Dockerfile + fi + sed -i "s/percona-xtrabackup-80/percona-xtrabackup-${PXC_MAJOR_RELEASE}/g" Dockerfile + fi + sudo docker build --no-cache --platform "linux/amd64" -t perconalab/percona-xtradb-cluster-operator:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}-pxc8.${MYSQL_VERSION_MINOR}-backup . + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION + . ./MYSQL_VERSION + sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-amd64 + sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-arm64 + sudo docker push perconalab/percona-xtradb-cluster-operator:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}-pxc8.${MYSQL_VERSION_MINOR}-backup + ''' + } + sh ''' + curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION + . ./MYSQL_VERSION + sudo docker manifest create perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} \ + perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 \ + perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION + . ./MYSQL_VERSION + sudo docker manifest push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH} perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR} perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtradb-cluster:latest perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + ''' + } + } + } + } +stage('Check by trivy') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + environment { + TRIVY_LOG = "trivy-high-junit.xml" + } + steps { + script { + try { + echo "🔄 Fetching MySQL version..." + + // 🔹 Capture the file content directly from curl + def mysqlVersion = sh( + script: "curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION", + returnStdout: true + ).trim() + + echo "🔎 Raw MYSQL_VERSION: '${mysqlVersion}'" + + if (!mysqlVersion) { + error "❌ MYSQL_VERSION file is empty or not found!" + } + + def versionMap = [:] + mysqlVersion.split('\n').each { line -> + def (key, value) = line.tokenize('=') + versionMap[key.trim()] = value.trim().replaceAll('"', '') + } + + def MYSQL_VERSION_MAJOR = versionMap['MYSQL_VERSION_MAJOR'] + def MYSQL_VERSION_MINOR = versionMap['MYSQL_VERSION_MINOR'] + def MYSQL_VERSION_PATCH = versionMap['MYSQL_VERSION_PATCH'] + def MYSQL_VERSION_EXTRA = versionMap['MYSQL_VERSION_EXTRA'] + def fullVersion = "${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}" + + echo "✅ Parsed MySQL version: ${fullVersion}" + + // 🔹 Install Trivy if not already installed + sh ''' + if ! command -v trivy &> /dev/null; then + echo "🔄 Installing Trivy..." + sudo apt-get update + sudo apt-get -y install wget apt-transport-https gnupg lsb-release + wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - + echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list + sudo apt-get update + sudo apt-get -y install trivy + else + echo "✅ Trivy is already installed." + fi + ''' + + // 🔹 Define the image tags + def imageList = [ + "perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64", + "perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64" + ] + + // 🔹 Scan images and store logs + imageList.each { image -> + echo "🔍 Scanning ${image}..." + def result = sh(script: """#!/bin/bash + sudo trivy image --quiet \ + --format table \ + --timeout 10m0s \ + --ignore-unfixed \ + --exit-code 1 \ + --severity HIGH,CRITICAL ${image} | tee -a ${TRIVY_LOG} + """, returnStatus: true) + + if (result != 0) { + error "❌ Trivy detected vulnerabilities in ${image}. See ${TRIVY_LOG} for details." + } else { + echo "✅ No critical vulnerabilities found in ${image}." + } + } + } catch (Exception e) { + error "❌ Trivy scan failed: ${e.message}" + } // try + } // script + } // steps + } // stage +/* + stage('Check by trivy') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + steps { + catchError { + sh ''' + curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION + . ./MYSQL_VERSION + sudo apt-get update + sudo apt-get -y install wget apt-transport-https gnupg lsb-release + wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - + echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list + sudo apt-get update + sudo apt-get -y install trivy + sudo trivy -q image --format table \ + --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 | tee -a trivy-hight-junit.xml + sudo trivy -q image --format table \ + --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 | tee -a trivy-hight-junit.xml + ''' + } + } + } +*/ + } + post { + success { + script { + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH}") + } + deleteDir() + } + failure { + script { + slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH}]") + } + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + } +} diff --git a/pxc/jenkins/percona-xtradb-cluster-8.0-docker.yml b/pxc/jenkins/percona-xtradb-cluster-8.0-docker.yml new file mode 100644 index 0000000000..a4db063a58 --- /dev/null +++ b/pxc/jenkins/percona-xtradb-cluster-8.0-docker.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pxc8.0-docker-build + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/adivinho/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pxc/jenkins/percona-xtradb-cluster-8.0-docker.groovy diff --git a/pxc/jenkins/percona-xtradb-cluster-8.0.groovy b/pxc/jenkins/percona-xtradb-cluster-8.0.groovy index ddeec39b36..85719d0f81 100644 --- a/pxc/jenkins/percona-xtradb-cluster-8.0.groovy +++ b/pxc/jenkins/percona-xtradb-cluster-8.0.groovy @@ -1,22 +1,47 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ void buildStage(String DOCKER_OS, String STAGE_PARAM) { - sh """ - set -o xtrace - mkdir -p test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/build-ps/pxc_builder.sh -O pxc_builder.sh - pwd -P - ls -laR - export build_dir=\$(pwd -P) - docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " - set -o xtrace - cd \${build_dir} - bash -x ./pxc_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./pxc_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --bin_release=${BIN_RELEASE} ${STAGE_PARAM}" - """ + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + withCredentials([usernamePassword(credentialsId: 'PS_PRIVATE_REPO_ACCESS', passwordVariable: 'PASSWORD', usernameVariable: 'USERNAME')]) { + sh """ + set -o xtrace + mkdir -p test + if [ \${FIPSMODE} = "YES" ]; then + PXC_VERSION_MINOR=\$(curl -s -O \$(echo \${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/\${GIT_BRANCH}/MYSQL_VERSION && cat MYSQL_VERSION | grep MYSQL_VERSION_MINOR | awk -F= '{print \$2}') + if [ \${PXC_VERSION_MINOR} = "0" ]; then + PRO_BRANCH="8.0" + elif [ \${PXC_VERSION_MINOR} = "4" ]; then + PRO_BRANCH="8.4" + else + PRO_BRANCH="trunk" + fi + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o pxc_builder.sh \ + "https://api.github.com/repos/percona/percona-xtradb-cluster-private-build/contents/build-ps/pxc_builder.sh?ref=\${PRO_BRANCH}" + sed -i "s/PRIVATE_USERNAME/\${USERNAME}/g" pxc_builder.sh + sed -i "s/PRIVATE_TOKEN/\${PASSWORD}/g" pxc_builder.sh + grep "percona-release enable pxb-80-pro" pxc_builder.sh + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/build-ps/pxc_builder.sh -O pxc_builder.sh + fi + pwd -P + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./pxc_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FIPSMODE} = "YES" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/percona-xtradb-cluster-private-build.git percona-xtradb-cluster-private-build + mv -f \${build_dir}/percona-xtradb-cluster-private-build/build-ps \${build_dir}/test/. + fi + bash -x ./pxc_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --bin_release=${BIN_RELEASE} ${STAGE_PARAM}" + """ + } + } } void cleanUpWS() { @@ -29,9 +54,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-xtradb-cluster.git', description: 'URL for percona-xtradb-cluster repository', @@ -52,10 +81,6 @@ pipeline { defaultValue: '1', description: 'BIN release value', name: 'BIN_RELEASE') - choice( - choices: 'pxc-80\npxc-8x-innovation\npxc-84-lts', - description: 'PXC repo name', - name: 'PXC_REPO') choice( choices: 'NO\nYES', description: 'Enable fipsmode', @@ -77,10 +102,19 @@ pipeline { } stages { stage('Create PXC source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } steps { slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") cleanUpWS() - buildStage("centos:7", "--get_sources=1") + script { + if (env.FIPSMODE == 'YES') { + buildStage("centos:7", "--get_sources=1 --enable_fipsmode=1") + } else { + buildStage("centos:7", "--get_sources=1") + } + } sh ''' REPO_UPLOAD_PATH=$(grep "DEST=UPLOAD" test/pxc-80.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -95,20 +129,20 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PXC generic source packages') { parallel { stage('Build PXC generic source rpm') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("centos:7", "--build_src_rpm=1 --enable_fipsmode=1") @@ -117,18 +151,18 @@ pipeline { } } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PXC generic source deb') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:xenial", "--build_source_deb=1 --enable_fipsmode=1") @@ -137,35 +171,17 @@ pipeline { } } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PXC RPMs/DEBs/Binary tarballs') { parallel { - stage('Centos 7') { - agent { - label 'docker-32gb' - } - steps { - echo "The step is skipped" -/* - cleanUpWS() - unstash 'pxc-80.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("centos:7", "--build_rpm=1") - - stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) -*/ - } - } stage('Centos 8') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -174,19 +190,19 @@ pipeline { } else { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Centos 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -195,24 +211,24 @@ pipeline { } else { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Oracle Linux 9') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") @@ -221,18 +237,18 @@ pipeline { } } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") @@ -241,28 +257,55 @@ pipeline { } } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } -/* stage('Amazon Linux 2023') { + stage('Amazon Linux 2023') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { - cleanUpWS() - unstash 'pxc-80.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("amazonlinux:2023", "--build_rpm=1") + script { + if (env.FIPSMODE == 'NO') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") - stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Amazon Linux 2023 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + if (env.FIPSMODE == 'NO') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } } - }*/ + } stage('Ubuntu Focal(20.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -271,19 +314,19 @@ pipeline { } else { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -292,24 +335,24 @@ pipeline { } else { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") @@ -319,18 +362,18 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") @@ -340,18 +383,18 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") @@ -361,18 +404,18 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") @@ -382,13 +425,13 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -397,19 +440,19 @@ pipeline { } else { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -418,24 +461,24 @@ pipeline { } else { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Bookworm(12)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") @@ -445,18 +488,18 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") @@ -466,13 +509,13 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 8 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -481,24 +524,24 @@ pipeline { } else { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:8", "--build_tarball=1") stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } } stage('Oracle Linux 9 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") @@ -508,13 +551,13 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } stage('Debian Bullseye(11) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -523,24 +566,24 @@ pipeline { } else { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1") stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } } stage('Ubuntu Jammy(22.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") @@ -550,8 +593,8 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } @@ -570,22 +613,22 @@ pipeline { if ("${PXC_VERSION_MINOR}" == "0") { // sync packages if (env.FIPSMODE == 'YES') { - sync2PrivateProdAutoBuild("pxc-80-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-80-pro", COMPONENT) } else { - sync2ProdAutoBuild("pxc-80", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "pxc-80", COMPONENT) } } else { if (env.FIPSMODE == 'YES') { if ("${PXC_VERSION_MINOR}" == "4") { - sync2PrivateProdAutoBuild("pxc-84-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-84-pro", COMPONENT) } else { - sync2PrivateProdAutoBuild("pxc-8x-innovation-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-8x-innovation-pro", COMPONENT) } } else { if ("${PXC_VERSION_MINOR}" == "4") { - sync2ProdAutoBuild("pxc-84-lts", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "pxc-84-lts", COMPONENT) } else { - sync2ProdAutoBuild("pxc-8x-innovation", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "pxc-8x-innovation", COMPONENT) } } } @@ -597,9 +640,9 @@ pipeline { script { try { if (env.FIPSMODE == 'YES') { - uploadTarballToDownloadsTesting("pxc-gated", "${GIT_BRANCH}") + uploadTarballToDownloadsTesting(params.CLOUD, "pxc-gated", "${GIT_BRANCH}") } else { - uploadTarballToDownloadsTesting("pxc", "${GIT_BRANCH}") + uploadTarballToDownloadsTesting(params.CLOUD, "pxc", "${GIT_BRANCH}") } } catch (err) { @@ -611,7 +654,7 @@ pipeline { } stage('Build docker containers') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' } steps { script { @@ -627,31 +670,38 @@ pipeline { sh ''' PXC_RELEASE=$(echo ${GIT_BRANCH} | sed 's/release-//g') PXC_MAJOR_RELEASE=$(echo ${GIT_BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}') + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common - sudo apt-get install -y docker.io - sudo systemctl status docker - sudo apt-get install -y qemu binfmt-support qemu-user-static + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION . ./MYSQL_VERSION git clone https://github.com/percona/percona-docker - cd percona-docker/percona-xtradb-cluster-8.0 + cd percona-docker/percona-xtradb-cluster-${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR} sed -i "s/ENV PXC_VERSION.*/ENV PXC_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile sed -i "s/ENV PXC_TELEMETRY_VERSION.*/ENV PXC_TELEMETRY_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}-${RPM_RELEASE}/g" Dockerfile sed -i "s/ENV PXC_REPO .*/ENV PXC_REPO testing/g" Dockerfile if [ ${PXC_MAJOR_RELEASE} != "80" ]; then if [ ${PXC_MAJOR_RELEASE} != "84" ]; then sed -i "s/pxc-80/pxc-8x-innovation/g" Dockerfile - else - sed -i "s/pdpxc-8.0/pdpxc-84-lts/g" Dockerfile - sed -i "s/pxc-80/pxc-84-lts/g" Dockerfile - sed -i "s/default_authentication_plugin=mysql_native_password/mysql-native-password=ON\\nrequire_secure_transport=OFF/g" dockerdir/etc/mysql/node.cnf - sed -i "s/skip-host-cache/host_cache_size = 0/g" dockerdir/etc/mysql/node.cnf - sed -i "s/--skip-ssl//g" dockerdir/entrypoint.sh fi fi - sudo docker build --no-cache --platform "linux/amd64" -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} . - sudo docker build --no-cache --platform "linux/amd64" --build-arg DEBUG=1 -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug . + sudo docker build --no-cache --platform "linux/amd64" -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 . + sudo docker build --no-cache --platform "linux/amd64" --build-arg DEBUG=1 -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-amd64 . sed -i "s/ENV PXC_VERSION.*/ENV PXC_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile.aarch64 sed -i "s/ENV PXC_TELEMETRY_VERSION.*/ENV PXC_TELEMETRY_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}-${RPM_RELEASE}/g" Dockerfile.aarch64 @@ -659,16 +709,10 @@ pipeline { if [ ${PXC_MAJOR_RELEASE} != "80" ]; then if [ ${PXC_MAJOR_RELEASE} != "84" ]; then sed -i "s/pxc-80/pxc-8x-innovation/g" Dockerfile.aarch64 - else - sed -i "s/pdpxc-8.0/pdpxc-84-lts/g" Dockerfile.aarch64 - sed -i "s/pxc-80/pxc-84-lts/g" Dockerfile.aarch64 - sed -i "s/default_authentication_plugin=mysql_native_password/mysql-native-password=ON\\nrequire_secure_transport=OFF/g" dockerdir/etc/mysql/node.cnf - sed -i "s/skip-host-cache/host_cache_size = 0/g" dockerdir/etc/mysql/node.cnf - sed -i "s/--skip-ssl//g" dockerdir/entrypoint.sh fi fi - sudo docker build --no-cache -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-aarch64 --platform="linux/arm64" -f Dockerfile.aarch64 . - sudo docker build --no-cache --build-arg DEBUG=1 -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-aarch64 --platform="linux/arm64" -f Dockerfile.aarch64 . + sudo docker build --no-cache -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --platform="linux/arm64" -f Dockerfile.aarch64 . + sudo docker build --no-cache --build-arg DEBUG=1 -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-arm64 --platform="linux/arm64" -f Dockerfile.aarch64 . if [ ${PXC_MAJOR_RELEASE} != "84" ]; then cd ../percona-xtradb-cluster-8.0-backup @@ -705,22 +749,22 @@ pipeline { echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION . ./MYSQL_VERSION - sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} - sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug - sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-aarch64 - sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-aarch64 + sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-amd64 + sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-arm64 sudo docker push perconalab/percona-xtradb-cluster-operator:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}-pxc8.${MYSQL_VERSION_MINOR}-backup ''' } sh ''' curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION . ./MYSQL_VERSION - sudo docker manifest create perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-multi \ - perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} \ - perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-aarch64 - sudo docker manifest annotate perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-multi perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-aarch64 --os linux --arch arm64 --variant v8 - sudo docker manifest annotate perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-multi perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} --os linux --arch amd64 - sudo docker manifest inspect perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-multi + sudo docker manifest create perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} \ + perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 \ + perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} ''' withCredentials([ usernamePassword(credentialsId: 'hub.docker.com', @@ -731,7 +775,10 @@ pipeline { echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION . ./MYSQL_VERSION - sudo docker manifest push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-multi + sudo docker manifest push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH} perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR} perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtradb-cluster:latest perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} ''' } } diff --git a/pxc/jenkins/percona-xtradb-cluster-8.0.yml b/pxc/jenkins/percona-xtradb-cluster-8.0.yml index 87d235dcb2..cdf5b2bdfc 100644 --- a/pxc/jenkins/percona-xtradb-cluster-8.0.yml +++ b/pxc/jenkins/percona-xtradb-cluster-8.0.yml @@ -1,5 +1,5 @@ - job: - name: pxc80-autobuild-RELEASE + name: hetzner-pxc80-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pxc/jenkins/percona-xtradb-cluster-8.0.groovy diff --git a/pxc/jenkins/percona-xtradb-cluster-9.x.groovy b/pxc/jenkins/percona-xtradb-cluster-9.x.groovy new file mode 100644 index 0000000000..71062956be --- /dev/null +++ b/pxc/jenkins/percona-xtradb-cluster-9.x.groovy @@ -0,0 +1,733 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ + set -o xtrace + mkdir -p test + if [ \${FIPSMODE} = "YES" ]; then + PXC_VERSION_MINOR=\$(curl -s -O \$(echo \${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/\${GIT_BRANCH}/MYSQL_VERSION && cat MYSQL_VERSION | grep MYSQL_VERSION_MINOR | awk -F= '{print \$2}') + if [ \${PXC_VERSION_MINOR} = "0" ]; then + PRO_BRANCH="8.0" + elif [ \${PXC_VERSION_MINOR} = "4" ]; then + PRO_BRANCH="8.4" + else + PRO_BRANCH="trunk" + fi + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o pxc_builder.sh \ + "https://api.github.com/repos/percona/percona-xtradb-cluster-private-build/contents/build-ps/pxc_builder.sh?ref=\${PRO_BRANCH}" + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/build-ps/pxc_builder.sh -O pxc_builder.sh + fi + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./pxc_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FIPSMODE} = "YES" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/percona-xtradb-cluster-private-build.git percona-xtradb-cluster-private-build + mv -f \${build_dir}/percona-xtradb-cluster-private-build/build-ps \${build_dir}/test/. + fi + bash -x ./pxc_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --bin_release=${BIN_RELEASE} ${STAGE_PARAM}" + """ + } +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/percona-xtradb-cluster.git', + description: 'URL for percona-xtradb-cluster repository', + name: 'GIT_REPO') + string( + defaultValue: '9.0', + description: 'Tag/Branch for percona-xtradb-cluster repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: '1', + description: 'BIN release value', + name: 'BIN_RELEASE') + choice( + choices: 'pxc-9x-innovation\npxc-97-lts', + description: 'PXC repo name', + name: 'PXC_REPO') + choice( + choices: 'NO\nYES', + description: 'Enable fipsmode', + name: 'FIPSMODE') + choice( + choices: 'YES\nNO', + description: 'Experimental packages only', + name: 'EXPERIMENTALMODE') + choice( + choices: 'experimental\nlaboratory\ntesting', + description: 'Repo component to push packages to', + name: 'COMPONENT') + choice( + choices: '#releases-ci\n#releases', + description: 'Channel for notifications', + name: 'SLACKNOTIFY') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Create PXC source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + steps { + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + script { + if (env.FIPSMODE == 'YES') { + buildStage("centos:7", "--get_sources=1 --enable_fipsmode=1") + } else { + buildStage("centos:7", "--get_sources=1") + } + } + sh ''' + REPO_UPLOAD_PATH=$(grep "DEST=UPLOAD" test/pxc-9x.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pxc-9x.properties + cat uploadPath + cat awsUploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PXC generic source packages') { + parallel { + stage('Build PXC generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + script { + if (env.FIPSMODE == 'YES') { + buildStage("centos:7", "--build_src_rpm=1 --enable_fipsmode=1") + } else { + buildStage("centos:7", "--build_src_rpm=1") + } + } + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PXC generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + script { + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:xenial", "--build_source_deb=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:xenial", "--build_source_deb=1") + } + } + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PXC RPMs/DEBs/Binary tarballs') { + parallel { + stage('Centos 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("centos:8", "--build_rpm=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + } + stage('Centos 8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("centos:8", "--build_rpm=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") + } else { + buildStage("oraclelinux:9", "--build_rpm=1") + } + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Oracle Linux 9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") + } else { + buildStage("oraclelinux:9", "--build_rpm=1") + } + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Amazon Linux 2023') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FIPSMODE == 'NO') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + } + stage('Amazon Linux 2023 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + if (env.FIPSMODE == 'NO') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + } + stage('Ubuntu Focal(20.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + } + stage('Ubuntu Focal(20.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:jammy", "--build_deb=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:jammy", "--build_deb=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:noble", "--build_deb=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:noble", "--build_deb=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + } + stage('Debian Bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") + } else { + buildStage("debian:bookworm", "--build_deb=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + stage('Debian Bookworm(12) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") + } else { + buildStage("debian:bookworm", "--build_deb=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + stage('Centos 8 tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("centos:8", "--build_tarball=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') + } + } + } + } + } + stage('Oracle Linux 9 tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") + } else { + buildStage("oraclelinux:9", "--build_tarball=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') + } + } + } + } + stage('Debian Bullseye(11) tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_tarball=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') + } + } + } + } + } + stage('Ubuntu Jammy(22.04) tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + script { + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:jammy", "--build_tarball=1") + } + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') + } + } + } + } + + stage('Sign packages') { + steps { + script { + if (env.EXPERIMENTALMODE == 'NO') { + signRPM() + } + } + signDEB() + } + } + stage('Push to public repository') { + steps { + script { + PXC_VERSION_MINOR = sh(returnStdout: true, script: ''' curl -s -O $(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git$||')/${GIT_BRANCH}/MYSQL_VERSION; cat MYSQL_VERSION | grep MYSQL_VERSION_MINOR | awk -F= '{print $2}' ''').trim() + if ("${PXC_VERSION_MINOR}" == "0") { + // sync packages + if (env.FIPSMODE == 'YES') { + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-90-pro", COMPONENT) + } else { + sync2ProdAutoBuild(params.CLOUD, "pxc-90", COMPONENT) + } + } else { + if (env.FIPSMODE == 'YES') { + if ("${PXC_VERSION_MINOR}" == "7") { + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-97-pro", COMPONENT) + } else { + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-9x-innovation-pro", COMPONENT) + } + } else { + if ("${PXC_VERSION_MINOR}" == "7") { + sync2ProdAutoBuild(params.CLOUD, "pxc-97-lts", COMPONENT) + } else { + sync2ProdAutoBuild(params.CLOUD, "pxc-9x-innovation", COMPONENT) + } + } + } + } + } + } + stage('Push Tarballs to TESTING download area') { + steps { + script { + try { + if (env.FIPSMODE == 'YES') { + uploadTarballToDownloadsTesting(params.CLOUD, "pxc-gated", "${GIT_BRANCH}") + } else { + uploadTarballToDownloadsTesting(params.CLOUD, "pxc", "${GIT_BRANCH}") + } + } + catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } + } + } + } + post { + success { + script { + if (env.FIPSMODE == 'YES') { + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: PRO build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + } else { + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + } + } + deleteDir() + } + failure { + script { + if (env.FIPSMODE == 'YES') { + slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: PRO build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + } else { + slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + } + } + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + script { + if (env.FIPSMODE == 'YES') { + currentBuild.description = "PRO -> Built on ${GIT_BRANCH} - packages [${COMPONENT}/${AWS_STASH_PATH}]" + } else { + currentBuild.description = "Built on ${GIT_BRANCH} - packages [${COMPONENT}/${AWS_STASH_PATH}]" + } + } + deleteDir() + } + } +} diff --git a/pxc/jenkins/percona-xtradb-cluster-9.x.yml b/pxc/jenkins/percona-xtradb-cluster-9.x.yml new file mode 100644 index 0000000000..dadc4f5e76 --- /dev/null +++ b/pxc/jenkins/percona-xtradb-cluster-9.x.yml @@ -0,0 +1,15 @@ +- job: + name: pxc9x-autobuild-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pxc/jenkins/percona-xtradb-cluster-9.x.groovy + diff --git a/rel/jenkins/release-from-repository.groovy b/rel/jenkins/release-from-repository.groovy index b5635297f6..6707b703ec 100644 --- a/rel/jenkins/release-from-repository.groovy +++ b/rel/jenkins/release-from-repository.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,10 @@ pipeline { label 'jenkins' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: '', description: 'PATH_TO_BUILD must be in form $DESTINATION/**release**/$revision', diff --git a/rel/jenkins/release-from-repository.yml b/rel/jenkins/release-from-repository.yml index 7b2b8c73f4..1041a8f0d8 100644 --- a/rel/jenkins/release-from-repository.yml +++ b/rel/jenkins/release-from-repository.yml @@ -1,5 +1,5 @@ - job: - name: RELEASE-from-repository + name: hetzner-RELEASE-from-repository project-type: pipeline description: | Do not edit this job through the web! @@ -9,7 +9,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: rel/jenkins/release-from-repository.groovy diff --git a/sysbench/sysbench.groovy b/sysbench/sysbench.groovy index 9a8114cc8a..c87ccb7383 100644 --- a/sysbench/sysbench.groovy +++ b/sysbench/sysbench.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -28,9 +28,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/akopytov/sysbench.git', description: 'URL for sysbench repository', @@ -99,36 +103,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build Sysbench generic source packages') { parallel { stage('Build Sysbench generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build Sysbench generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -137,184 +141,184 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } diff --git a/sysbench/sysbench.yml b/sysbench/sysbench.yml index be75172fd6..1002ba764b 100644 --- a/sysbench/sysbench.yml +++ b/sysbench/sysbench.yml @@ -1,5 +1,5 @@ - job: - name: sysbench-RELEASE + name: hetzner-sysbench-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: sysbench/sysbench.groovy diff --git a/vars/popArtifactFolder.groovy b/vars/popArtifactFolder.groovy index 67e8f899f9..802192c497 100644 --- a/vars/popArtifactFolder.groovy +++ b/vars/popArtifactFolder.groovy @@ -1,10 +1,14 @@ -def call(String FOLDER_NAME, String AWS_STASH_PATH) { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sh """ - pwd - S3_PATH=s3://percona-jenkins-artifactory/${AWS_STASH_PATH} - aws s3 cp --recursive \$S3_PATH/${FOLDER_NAME} ${FOLDER_NAME} || : - """ +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH) { + String S3_STASH = (CLOUD_NAME == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (CLOUD_NAME == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + retry(15) { + sh """ + pwd + S3_PATH=s3://percona-jenkins-artifactory/${AWS_STASH_PATH} + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp --recursive \$S3_PATH/${FOLDER_NAME} ${FOLDER_NAME} ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 || : + """ + } } } diff --git a/vars/pushArtifactFolder.groovy b/vars/pushArtifactFolder.groovy index b80c26f29c..f1e8d93307 100644 --- a/vars/pushArtifactFolder.groovy +++ b/vars/pushArtifactFolder.groovy @@ -1,11 +1,15 @@ -def call(String FOLDER_NAME, String AWS_STASH_PATH) { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sh """ - pwd - S3_PATH=s3://percona-jenkins-artifactory/${AWS_STASH_PATH} - aws s3 ls \$S3_PATH/${FOLDER_NAME} || : - aws s3 cp --quiet --recursive ${FOLDER_NAME} \$S3_PATH/${FOLDER_NAME} || : - """ +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH) { + String S3_STASH = (CLOUD_NAME == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (CLOUD_NAME == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + retry(15) { + sh """ + pwd + S3_PATH=s3://percona-jenkins-artifactory/${AWS_STASH_PATH} + aws s3 ls \$S3_PATH/${FOLDER_NAME} ${S3_ENDPOINT} || : + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp --recursive ${FOLDER_NAME} \$S3_PATH/${FOLDER_NAME} ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 || : + """ + } } } diff --git a/vars/signDEB.groovy b/vars/signDEB.groovy index 865cc734b2..0836f6f9ed 100644 --- a/vars/signDEB.groovy +++ b/vars/signDEB.groovy @@ -1,5 +1,6 @@ -def call() { - node('master') { +def call(String CLOUD_NAME = 'default') { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { unstash 'uploadPath' withCredentials([string(credentialsId: 'SIGN_PASSWORD', variable: 'SIGN_PASSWORD')]) { withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', passphraseVariable: '', usernameVariable: 'USER')]) { diff --git a/vars/signRPM.groovy b/vars/signRPM.groovy index 042a9d6450..df9ce77952 100644 --- a/vars/signRPM.groovy +++ b/vars/signRPM.groovy @@ -1,5 +1,6 @@ -def call() { - node('master') { +def call(String CLOUD_NAME = 'default') { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { unstash 'uploadPath' withCredentials([string(credentialsId: 'SIGN_PASSWORD', variable: 'SIGN_PASSWORD')]) { withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', passphraseVariable: '', usernameVariable: 'USER')]) { diff --git a/vars/sync2PrivateProdAutoBuild.groovy b/vars/sync2PrivateProdAutoBuild.groovy index 9c676d0a84..82aba5bae5 100644 --- a/vars/sync2PrivateProdAutoBuild.groovy +++ b/vars/sync2PrivateProdAutoBuild.groovy @@ -1,5 +1,6 @@ -def call(String REPO_NAME, String DESTINATION) { - node('master') { +def call(String CLOUD_NAME, String REPO_NAME, String DESTINATION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { unstash 'uploadPath' def path_to_build = sh(returnStdout: true, script: "cat uploadPath").trim() diff --git a/vars/sync2ProdAutoBuild.groovy b/vars/sync2ProdAutoBuild.groovy index e726b0db3b..62f5537065 100644 --- a/vars/sync2ProdAutoBuild.groovy +++ b/vars/sync2ProdAutoBuild.groovy @@ -1,5 +1,6 @@ -def call(String REPO_NAME, String DESTINATION) { - node('master') { +def call(String CLOUD_NAME, String REPO_NAME, String DESTINATION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { unstash 'uploadPath' def path_to_build = sh(returnStdout: true, script: "cat uploadPath").trim() diff --git a/vars/sync2ProdPPG.groovy b/vars/sync2ProdPPG.groovy index 1d8f15b61a..680b7daca0 100644 --- a/vars/sync2ProdPPG.groovy +++ b/vars/sync2ProdPPG.groovy @@ -1,5 +1,6 @@ -def call(String REPO_NAME, String DESTINATION) { - node('master') { +def call(String CLOUD_NAME, String REPO_NAME, String DESTINATION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { def path_to_build = "/srv/UPLOAD/POSTGRESQL_SYNC/${repo_version}" withCredentials([string(credentialsId: 'SIGN_PASSWORD', variable: 'SIGN_PASSWORD')]) { withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', passphraseVariable: '', usernameVariable: 'USER')]) { diff --git a/vars/sync2web.groovy b/vars/sync2web.groovy index 101e701dd0..866e2a44b9 100644 --- a/vars/sync2web.groovy +++ b/vars/sync2web.groovy @@ -1,5 +1,6 @@ -def call(String REPO_NAME, String DESTINATION) { - node('master') { +def call(String CLOUD_NAME, String REPO_NAME, String DESTINATION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { deleteDir() unstash 'uploadPath' def path_to_build = "/srv/UPLOAD/POSTGRESQL_SYNC/${repo_version}" diff --git a/vars/uploadDEBfromAWS.groovy b/vars/uploadDEBfromAWS.groovy index d5917e793a..9493feb9ea 100644 --- a/vars/uploadDEBfromAWS.groovy +++ b/vars/uploadDEBfromAWS.groovy @@ -1,7 +1,8 @@ -def call(String FOLDER_NAME, String AWS_STASH_PATH) { - node('master') { +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { deleteDir() - popArtifactFolder(FOLDER_NAME, AWS_STASH_PATH) + popArtifactFolder(CLOUD_NAME, FOLDER_NAME, AWS_STASH_PATH) //unstash 'debs' unstash 'uploadPath' withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { @@ -25,7 +26,12 @@ def call(String FOLDER_NAME, String AWS_STASH_PATH) { dist=`echo \${deb} | sed -re 's/.*\\.([^.]+)_(amd64|arm64|all).deb/\\1/'` package=`echo \${deb} | sed -re 's/\\.\\/deb\\/(.*)_(.*)\\.([^.]+)_(amd64|arm64|all).deb/\\1/'` version=`echo \${deb} | sed -re 's/\\.\\/deb\\/(.*)_(.*)\\.([^.]+)_(amd64|arm64|all).deb/\\2/'` - path_to_dist=\${path_to_build}/binary/debian/\${dist}/x86_64 + arch=`echo \${deb} | sed -re 's/\\.\\/deb\\/(.*)_(.*)\\.([^.]+)_(amd64|arm64|all).deb/\\4/'` + if [ "\${arch}" = "arm64" ]; then + path_to_dist=\${path_to_build}/binary/debian/\${dist}/aarch64 + else + path_to_dist=\${path_to_build}/binary/debian/\${dist}/x86_64 + fi ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ mkdir -p \${path_to_dist} if [ "\${package}" = "percona-release" ]; then diff --git a/vars/uploadPGSBOMToDownloadsTesting.groovy b/vars/uploadPGSBOMToDownloadsTesting.groovy new file mode 100644 index 0000000000..b1c286bb98 --- /dev/null +++ b/vars/uploadPGSBOMToDownloadsTesting.groovy @@ -0,0 +1,29 @@ +def call(String CLOUD_NAME, String PRODUCT_NAME, String PRODUCT_VERSION, String SBOMType) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'master' + node(nodeLabel) { + deleteDir() + unstash "uploadPath-${PRODUCT_VERSION}" + def path_to_build = sh(returnStdout: true, script: "cat uploadPath-${PRODUCT_VERSION}").trim() + withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { + sh """ + set -o xtrace + + cat /etc/hosts > hosts + echo '10.30.6.9 repo.ci.percona.com' >> hosts + sudo cp ./hosts /etc || true + + # Cut prefix if it's provided + cutProductVersion=\$(echo ${PRODUCT_VERSION} | sed 's/release-//g'); + + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + ssh -p 2222 jenkins-deploy.jenkins-deploy.web.r.int.percona.com mkdir -p /data/downloads/TESTING/${PRODUCT_NAME}-\${cutProductVersion} + + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + rsync -avt -e '"ssh -p 2222"' --bwlimit=50000 --progress ${path_to_build}/${SBOMType}/* jenkins-deploy.jenkins-deploy.web.r.int.percona.com:/data/downloads/TESTING/${PRODUCT_NAME}-\${cutProductVersion}/ + + curl https://www.percona.com/admin/config/percona/percona_downloads/crawl_directory + """ + } + deleteDir() + } +} diff --git a/vars/uploadPGSBOMfromAWS.groovy b/vars/uploadPGSBOMfromAWS.groovy new file mode 100644 index 0000000000..289a85ef03 --- /dev/null +++ b/vars/uploadPGSBOMfromAWS.groovy @@ -0,0 +1,26 @@ +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH, String SBOMType, String PRODUCT_VERSION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { + deleteDir() + popArtifactFolder(CLOUD_NAME, FOLDER_NAME, AWS_STASH_PATH) + unstash "uploadPath-${PRODUCT_VERSION}" + withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { + sh """ + export path_to_build=`cat uploadPath-${PRODUCT_VERSION}` + + cat /etc/hosts > hosts + echo '10.30.6.9 repo.ci.percona.com' >> hosts + sudo cp ./hosts /etc || true + + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + mkdir -p \${path_to_build}/${SBOMType}/ + + scp -o StrictHostKeyChecking=no -i ${KEY_PATH} \ + `find . -name '*.json'` \ + ${USER}@repo.ci.percona.com:\${path_to_build}/${SBOMType}/ + + """ + } + deleteDir() + } +} diff --git a/vars/uploadPGTarballToDownloadsTesting.groovy b/vars/uploadPGTarballToDownloadsTesting.groovy index 13d4dd15a0..153d817fb9 100644 --- a/vars/uploadPGTarballToDownloadsTesting.groovy +++ b/vars/uploadPGTarballToDownloadsTesting.groovy @@ -1,5 +1,6 @@ -def call(String PRODUCT_NAME, String PRODUCT_VERSION) { - node('master') { +def call(String CLOUD_NAME, String PRODUCT_NAME, String PRODUCT_VERSION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'master' + node(nodeLabel) { deleteDir() unstash "uploadPath-${PRODUCT_VERSION}" def path_to_build = sh(returnStdout: true, script: "cat uploadPath-${PRODUCT_VERSION}").trim() diff --git a/vars/uploadPGTarballfromAWS.groovy b/vars/uploadPGTarballfromAWS.groovy index 2aeafbccf4..0404c4d781 100644 --- a/vars/uploadPGTarballfromAWS.groovy +++ b/vars/uploadPGTarballfromAWS.groovy @@ -1,7 +1,8 @@ -def call(String FOLDER_NAME, String AWS_STASH_PATH, String TarballType, String PRODUCT_VERSION) { - node('master') { +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH, String TarballType, String PRODUCT_VERSION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { deleteDir() - popArtifactFolder(FOLDER_NAME, AWS_STASH_PATH) + popArtifactFolder(CLOUD_NAME, FOLDER_NAME, AWS_STASH_PATH) //unstash "${TarballType}.tarball" unstash "uploadPath-${PRODUCT_VERSION}" withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { diff --git a/vars/uploadRPMfromAWS.groovy b/vars/uploadRPMfromAWS.groovy index f4ace39139..fefa014111 100644 --- a/vars/uploadRPMfromAWS.groovy +++ b/vars/uploadRPMfromAWS.groovy @@ -1,7 +1,8 @@ -def call(String FOLDER_NAME, String AWS_STASH_PATH) { - node('master') { +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { deleteDir() - popArtifactFolder(FOLDER_NAME, AWS_STASH_PATH) + popArtifactFolder(CLOUD_NAME, FOLDER_NAME, AWS_STASH_PATH) unstash 'uploadPath' withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { sh """ @@ -63,8 +64,7 @@ def call(String FOLDER_NAME, String AWS_STASH_PATH) { fi if [ `find . -name "*.noarch.rpm" | wc -l` -gt 0 ]; then - Vers=("6" "7" "8" "9" "2023") - for osVer in "\${Vers[@]}"; do + for osVer in 6 7 8 9 2023; do ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ mkdir -p \${path_to_build}/binary/redhat/\${osVer}/\${arch} scp -o StrictHostKeyChecking=no -i ${KEY_PATH} \ diff --git a/vars/uploadTarballToDownloadsTesting.groovy b/vars/uploadTarballToDownloadsTesting.groovy index f31b52deae..89502eb4b1 100644 --- a/vars/uploadTarballToDownloadsTesting.groovy +++ b/vars/uploadTarballToDownloadsTesting.groovy @@ -1,10 +1,12 @@ -def call(String PRODUCT_NAME, String PRODUCT_VERSION) { - node('master') { +def call(String CLOUD_NAME, String PRODUCT_NAME, String PRODUCT_VERSION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'master' + node(nodeLabel) { deleteDir() unstash 'uploadPath' def path_to_build = sh(returnStdout: true, script: "cat uploadPath").trim() withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { sh """ + #!/bin/bash set -o xtrace cat /etc/hosts > hosts @@ -15,23 +17,16 @@ def call(String PRODUCT_NAME, String PRODUCT_VERSION) { cutProductVersion=\$(echo ${PRODUCT_VERSION} | sed 's/release-//g'); # Get PXC version from a file - if [ "x${PRODUCT_NAME}" == "xpxc" ]; then - if [[ "${PRODUCT_VERSION}" != *CUSTOM* ]]; then - curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${PRODUCT_VERSION}/MYSQL_VERSION - MYSQL_VERSION_MAJOR=\$(cat MYSQL_VERSION | grep MYSQL_VERSION_MAJOR | awk -F= '{print \$2}') - MYSQL_VERSION_MINOR=\$(cat MYSQL_VERSION | grep MYSQL_VERSION_MINOR | awk -F= '{print \$2}') - MYSQL_VERSION_PATCH=\$(cat MYSQL_VERSION | grep MYSQL_VERSION_PATCH | awk -F= '{print \$2}') - cutProductVersion=\${MYSQL_VERSION_MAJOR}.\${MYSQL_VERSION_MINOR}.\${MYSQL_VERSION_PATCH} - fi + if [ \"x${PRODUCT_NAME}\" = \"xpxc\" ]; then + curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${PRODUCT_VERSION}/MYSQL_VERSION + MYSQL_VERSION_MAJOR=\$(cat MYSQL_VERSION | grep MYSQL_VERSION_MAJOR | awk -F= '{print \$2}') + MYSQL_VERSION_MINOR=\$(cat MYSQL_VERSION | grep MYSQL_VERSION_MINOR | awk -F= '{print \$2}') + MYSQL_VERSION_PATCH=\$(cat MYSQL_VERSION | grep MYSQL_VERSION_PATCH | awk -F= '{print \$2}') + cutProductVersion=\${MYSQL_VERSION_MAJOR}.\${MYSQL_VERSION_MINOR}.\${MYSQL_VERSION_PATCH} fi - - if [[ "${PRODUCT_NAME}" != *gated* ]]; then - ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ - ssh -p 2222 jenkins-deploy.jenkins-deploy.web.r.int.percona.com mkdir -p /data/downloads/TESTING/${PRODUCT_NAME}-\${cutProductVersion} - - ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ - rsync -avt -e '"ssh -p 2222"' --bwlimit=50000 --progress ${path_to_build}/binary/tarball/* jenkins-deploy.jenkins-deploy.web.r.int.percona.com:/data/downloads/TESTING/${PRODUCT_NAME}-\${cutProductVersion}/ - else + echo \${cutProductVersion} + if expr \"${PRODUCT_NAME}\" : \".*gated.*\" > /dev/null; then + echo "Processing gated product" ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ mkdir -p /srv/repo-copy/private/qa-test/${PRODUCT_NAME}-\${cutProductVersion} @@ -40,6 +35,13 @@ def call(String PRODUCT_NAME, String PRODUCT_VERSION) { ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ rsync -avt --delete --delete-excluded --delete-after --progress --exclude=*.sh --exclude=*.bak /srv/repo-copy/private/qa-test/* 10.30.9.32:/www/repo.percona.com/htdocs/private/qa-test/ + else + echo "Processing non-gated product" + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + ssh -p 2222 jenkins-deploy.jenkins-deploy.web.r.int.percona.com mkdir -p /data/downloads/TESTING/${PRODUCT_NAME}-\${cutProductVersion} + + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + rsync -avt -e '"ssh -p 2222"' --bwlimit=50000 --progress ${path_to_build}/binary/tarball/* jenkins-deploy.jenkins-deploy.web.r.int.percona.com:/data/downloads/TESTING/${PRODUCT_NAME}-\${cutProductVersion}/ fi curl https://www.percona.com/admin/config/percona/percona_downloads/crawl_directory diff --git a/vars/uploadTarballfromAWS.groovy b/vars/uploadTarballfromAWS.groovy index 4c9d363c55..f941cf26e6 100644 --- a/vars/uploadTarballfromAWS.groovy +++ b/vars/uploadTarballfromAWS.groovy @@ -1,7 +1,8 @@ -def call(String FOLDER_NAME, String AWS_STASH_PATH, String TarballType) { - node('master') { +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH, String TarballType) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { deleteDir() - popArtifactFolder(FOLDER_NAME, AWS_STASH_PATH) + popArtifactFolder(CLOUD_NAME, FOLDER_NAME, AWS_STASH_PATH) //unstash "${TarballType}.tarball" unstash 'uploadPath' withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) {