From fc9eb556dc101340f98784536ccf35bd98fda638 Mon Sep 17 00:00:00 2001 From: Jon Roelofs Date: Fri, 1 Aug 2025 17:36:38 -0700 Subject: [PATCH] [GreenDragon] Duplicate and templatize clang-stage2-Rthinlto for bisection The new approach modularizes and templatizes the job so that we can automatically bisect build failures down to the commit responsible for them. The new jobs written in this style use the name-inspecting symlink trick that clang itself uses to determine driver mode, for organizational simplicity and deduplication of configuration. For now, I am duplicating the job wholesale, until we're comfortable with replacing all of the existing clang-* jobs with this style. As part of this, we're adding a templatized bisection job and corresponding runner. Currently, these cooperate to bisect the clang-stage2-Rthinlto-v2 job's failures, but going forward we should be set up to re-use that for any other job's bisection needs. We will also be well set up to add a test bisection runner, which could leverage the artifacts saved to S3 as a caching mechanism to speed up the search. --- zorg/jenkins/jobs/jobs/build-bisect | 70 +++ zorg/jenkins/jobs/jobs/build-bisect-run | 106 ++++ .../jobs/jobs/clang-stage2-Rthinlto-v2 | 1 + .../jobs/jobs/templated-clang-job.groovy | 49 ++ zorg/jenkins/lib/builders/ClangBuilder.groovy | 546 ++++++++++++++++++ .../templates/clang-stage2-Rthinlto.groovy | 41 ++ zorg/jenkins/lib/utils/BisectionUtils.groovy | 200 +++++++ 7 files changed, 1013 insertions(+) create mode 100644 zorg/jenkins/jobs/jobs/build-bisect create mode 100644 zorg/jenkins/jobs/jobs/build-bisect-run create mode 120000 zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto-v2 create mode 100644 zorg/jenkins/jobs/jobs/templated-clang-job.groovy create mode 100644 zorg/jenkins/lib/builders/ClangBuilder.groovy create mode 100644 zorg/jenkins/lib/templates/clang-stage2-Rthinlto.groovy create mode 100644 zorg/jenkins/lib/utils/BisectionUtils.groovy diff --git a/zorg/jenkins/jobs/jobs/build-bisect b/zorg/jenkins/jobs/jobs/build-bisect new file mode 100644 index 000000000..5bfe79e90 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/build-bisect @@ -0,0 +1,70 @@ +#!/usr/bin/env groovy +@Library('llvm-jenkins-lib') _ + +// Bisection orchestrator that manages the binary search process +// Uses build-bisect-run jobs to do the actual testing work +def bisectionUtils = evaluate readTrusted('zorg/jenkins/lib/utils/BisectionUtils.groovy') + +pipeline { + options { + disableConcurrentBuilds() + } + + parameters { + string(name: 'LABEL', defaultValue: params.LABEL ?: 'macos-x86_64', description: 'Node label to run on') + string(name: 'BISECT_GOOD', defaultValue: params.BISECT_GOOD ?: '', description: 'Known good commit') + string(name: 'BISECT_BAD', defaultValue: params.BISECT_BAD ?: '', description: 'Known bad commit') + + // Job template configuration to pass through to build-bisect-run + string(name: 'JOB_TEMPLATE', defaultValue: params.JOB_TEMPLATE ?: 'clang-stage2-Rthinlto', description: 'Job template to bisect') + string(name: 'BUILD_CONFIG', defaultValue: params.BUILD_CONFIG ?: '{}', description: 'Build configuration JSON') + string(name: 'ARTIFACT', defaultValue: params.ARTIFACT ?: 'llvm.org/clang-stage1-RA/latest', description: 'Base artifact to use') + } + + agent { + node { + label params.LABEL + } + } + + stages { + stage('Setup') { + steps { + script { + if (!params.BISECT_GOOD || !params.BISECT_BAD || !params.JOB_TEMPLATE) { + error "BISECT_GOOD, BISECT_BAD, and JOB_TEMPLATE parameters are required" + } + + echo "Starting bisection of ${params.JOB_TEMPLATE}: ${params.BISECT_GOOD}...${params.BISECT_BAD}" + echo "Build Config: ${params.BUILD_CONFIG}" + + // Set build description + currentBuild.description = "🔍 BISECTING [${params.JOB_TEMPLATE}]: ${params.BISECT_GOOD.take(8)}..${params.BISECT_BAD.take(8)}" + + // Clone the repository to get commit information + def clangBuilder = evaluate readTrusted('zorg/jenkins/lib/builders/ClangBuilder.groovy') + clangBuilder.checkoutStage() + } + } + } + + stage('Perform Bisection') { + steps { + script { + def failingCommit = bisectionUtils.performBisectionWithRunner( + params.BISECT_GOOD, + params.BISECT_BAD, + params.JOB_TEMPLATE, + params.BUILD_CONFIG, + params.ARTIFACT + ) + + bisectionUtils.reportBisectionResult(failingCommit, params.JOB_TEMPLATE) + + // Update build description with result + currentBuild.description = "✅ BISECTED [${params.JOB_TEMPLATE}]: failing commit ${failingCommit.take(8)}" + } + } + } + } +} \ No newline at end of file diff --git a/zorg/jenkins/jobs/jobs/build-bisect-run b/zorg/jenkins/jobs/jobs/build-bisect-run new file mode 100644 index 000000000..70d0caccc --- /dev/null +++ b/zorg/jenkins/jobs/jobs/build-bisect-run @@ -0,0 +1,106 @@ +#!/usr/bin/env groovy + +// Generic build runner that can execute any job template configuration +// This job handles the actual building/testing work for bisection +def clangBuilder = evaluate readTrusted('zorg/jenkins/lib/builders/ClangBuilder.groovy') + +pipeline { + options { + disableConcurrentBuilds() + } + + parameters { + string(name: 'LABEL', defaultValue: params.LABEL ?: 'macos-x86_64', description: 'Node label to run on') + string(name: 'GIT_SHA', defaultValue: params.GIT_SHA ?: '*/main', description: 'Git commit to build.') + string(name: 'ARTIFACT', defaultValue: params.ARTIFACT ?: 'llvm.org/clang-stage1-RA/latest', description: 'Clang artifact to use') + + // Job template configuration + string(name: 'JOB_TEMPLATE', defaultValue: params.JOB_TEMPLATE ?: 'clang-stage2-Rthinlto', description: 'Job template to use') + string(name: 'BUILD_CONFIG', defaultValue: params.BUILD_CONFIG ?: '{}', description: 'Build configuration JSON') + + // Bisection context (for build description) + string(name: 'BISECT_GOOD', defaultValue: params.BISECT_GOOD ?: '', description: 'Good commit for bisection context') + string(name: 'BISECT_BAD', defaultValue: params.BISECT_BAD ?: '', description: 'Bad commit for bisection context') + } + + agent { + node { + label params.LABEL + } + } + + stages { + stage('Setup Build Description') { + steps { + script { + def commitInfo = params.GIT_SHA.take(8) + def template = params.JOB_TEMPLATE + + if (params.BISECT_GOOD && params.BISECT_BAD) { + def goodShort = params.BISECT_GOOD.take(8) + def badShort = params.BISECT_BAD.take(8) + currentBuild.description = "🔍 BISECT RUN [${template}]: ${commitInfo} (${goodShort}..${badShort})" + } else { + currentBuild.description = "🔧 BUILD RUN [${template}]: ${commitInfo}" + } + + echo "Job Template: ${template}" + echo "Testing commit: ${commitInfo}" + echo "Build Config: ${params.BUILD_CONFIG}" + } + } + } + + stage('Checkout') { + steps { + script { + clangBuilder.checkoutStage() + } + } + } + + stage('Setup Venv') { + steps { + script { + clangBuilder.setupVenvStage() + } + } + } + + stage('Fetch Artifact') { + steps { + script { + clangBuilder.fetchArtifactStage() + } + } + } + + stage('Build') { + steps { + script { + // Load the job template dynamically + def template = evaluate readTrusted("zorg/jenkins/lib/templates/${params.JOB_TEMPLATE}.groovy") + + // Parse user build configuration + def userBuildConfig = [:] + if (params.BUILD_CONFIG && params.BUILD_CONFIG != '{}') { + userBuildConfig = readJSON text: params.BUILD_CONFIG + } + + // Apply template defaults with user overrides + def buildConfig = template.getDefaultBuildConfig(userBuildConfig) + + clangBuilder.buildStage(buildConfig) + } + } + } + } + + post { + always { + script { + clangBuilder.cleanupStage() + } + } + } +} \ No newline at end of file diff --git a/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto-v2 b/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto-v2 new file mode 120000 index 000000000..220028ad2 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto-v2 @@ -0,0 +1 @@ +templated-clang-job.groovy \ No newline at end of file diff --git a/zorg/jenkins/jobs/jobs/templated-clang-job.groovy b/zorg/jenkins/jobs/jobs/templated-clang-job.groovy new file mode 100644 index 000000000..d628126f9 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/templated-clang-job.groovy @@ -0,0 +1,49 @@ +#!/usr/bin/env groovy + +/* + * GENERIC TEMPLATED CLANG JOB + * + * This is a generic job script that automatically configures itself based on the Jenkins job name. + * It works by using symlinks - each job is just a symlink to this file with the appropriate name. + * + * NAMING CONVENTION: + * - Job names follow the pattern: clang-stage[N]-[CONFIG] + * - Template name is derived by stripping any version suffix (e.g., -v2) + * - Each template defines its own bisection policy in getJobConfig() + * + * EXAMPLES: + * clang-stage2-Rthinlto-v2 → template: clang-stage2-Rthinlto, bisection: per template + * clang-stage1-RA → template: clang-stage1-RA, bisection: per template + * clang-stage2-cmake-RgSan → template: clang-stage2-cmake-RgSan, bisection: per template + * + * TEMPLATE RESOLUTION: + * - Templates are loaded from zorg/jenkins/lib/templates/[JOB_TEMPLATE].groovy + * - Template defines build configuration (stage, cmake_type, projects, etc.) + * - Template defines job configuration (bisection policy, etc.) via getJobConfig() + * + * TO ADD A NEW JOB: + * 1. Create the template file: zorg/jenkins/lib/templates/your-job-pattern.groovy + * 2. Create symlink: ln -s templated-clang-job.groovy your-job-name + * 3. Done! The job will automatically use the correct template and settings. + * + * BISECTION: + * - Each template decides its own bisection policy in getJobConfig() + * - ThinLTO jobs enable bisection (useful for performance regressions) + * - Stage1 jobs disable bisection (failures often environmental) + * - Future templates can define custom bisection logic + */ + +def clangBuilder = evaluate readTrusted('zorg/jenkins/lib/builders/ClangBuilder.groovy') + +// Auto-configure based on Jenkins job name +def jobName = env.JOB_NAME ?: 'unknown' + +// Derive template name by stripping -v2 suffix if present +def templateName = jobName.replaceAll(/-v\d+$/, '') + +// Load the template and get its job configuration +def template = evaluate readTrusted("zorg/jenkins/lib/templates/${templateName}.groovy") +def jobConfig = template.getJobConfig(jobName) + +// Instantiate the templated pipeline +clangBuilder.createTemplatedPipeline(jobConfig).call() \ No newline at end of file diff --git a/zorg/jenkins/lib/builders/ClangBuilder.groovy b/zorg/jenkins/lib/builders/ClangBuilder.groovy new file mode 100644 index 000000000..7c81165ca --- /dev/null +++ b/zorg/jenkins/lib/builders/ClangBuilder.groovy @@ -0,0 +1,546 @@ +#!/usr/bin/env groovy + +class ClangBuilder { + + static def pipeline(config) { + def buildConfig = config.config ?: [:] + def stagesToRun = config.stages ?: ['checkout', 'build', 'test'] + def postFailureConfig = config.post_failure ?: [:] + + pipeline { + options { + disableConcurrentBuilds() + } + + parameters { + string(name: 'LABEL', defaultValue: params.LABEL ?: 'macos-x86_64', description: 'Node label to run on') + string(name: 'GIT_SHA', defaultValue: params.GIT_REVISION ?: '*/main', description: 'Git commit to build.') + string(name: 'ARTIFACT', defaultValue: params.ARTIFACT ?: 'llvm.org/clang-stage1-RA/latest', description: 'Clang artifact to use') + string(name: 'BISECT_GOOD', defaultValue: params.BISECT_GOOD ?: '', description: 'Good commit for bisection') + string(name: 'BISECT_BAD', defaultValue: params.BISECT_BAD ?: '', description: 'Bad commit for bisection') + booleanParam(name: 'IS_BISECT_JOB', defaultValue: params.IS_BISECT_JOB ?: false, description: 'Whether this is a bisection job') + } + + agent { + node { + label params.LABEL + } + } + + stages { + stage('Checkout') { + when { + expression { 'checkout' in stagesToRun } + } + steps { + script { + ClangBuilder.checkoutStage() + } + } + } + + stage('Setup Venv') { + when { + expression { 'checkout' in stagesToRun } + } + steps { + script { + ClangBuilder.setupVenvStage() + } + } + } + + stage('Fetch Artifact') { + when { + expression { 'build' in stagesToRun } + } + steps { + script { + ClangBuilder.fetchArtifactStage() + } + } + } + + stage('Build') { + when { + expression { 'build' in stagesToRun } + } + steps { + script { + ClangBuilder.buildStage(buildConfig) + } + } + } + + stage('Test') { + when { + expression { 'test' in stagesToRun } + } + steps { + script { + ClangBuilder.testStage() + } + } + post { + always { + script { + junit "clang-build/**/testresults.xunit.xml" + } + } + } + } + } + + post { + always { + script { + sh "rm -rf clang-build clang-install host-compiler *.tar.gz" + } + } + failure { + script { + // Only trigger bisection for main jobs, not bisection jobs themselves + if (!params.IS_BISECT_JOB && shouldTriggerBisection(postFailureConfig)) { + triggerBisection(config.name, postFailureConfig) + } + } + } + } + } + } + + static def checkoutStage() { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], extensions: [ + [$class: 'CloneOption', timeout: 30] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], extensions: [ + [$class: 'CloneOption', reference: '/Users/Shared/llvm-zorg.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ]]) + } + } + + static def setupVenvStage() { + withEnv(["PATH=$PATH:/usr/bin:/usr/local/bin"]) { + sh ''' + # Non-incremental, so always delete. + rm -rf clang-build clang-install host-compiler *.tar.gz + rm -rf venv + python3 -m venv venv + set +u + source ./venv/bin/activate + pip install -r ./llvm-zorg/zorg/jenkins/jobs/requirements.txt + set -u + ''' + } + } + + static def fetchArtifactStage() { + withEnv(["PATH=$PATH:/usr/bin:/usr/local/bin"]) { + withCredentials([string(credentialsId: 's3_resource_bucket', variable: 'S3_BUCKET')]) { + sh """ + source ./venv/bin/activate + echo "ARTIFACT=${params.ARTIFACT}" + python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + ls $WORKSPACE/host-compiler/lib/clang/ + VERSION=`ls $WORKSPACE/host-compiler/lib/clang/` + """ + } + } + } + + static def buildStage(config = [:]) { + def thinlto = config.thinlto ?: false + def cmakeType = config.cmake_type ?: "RelWithDebInfo" + def projects = config.projects ?: "clang;clang-tools-extra;compiler-rt" + def runtimes = config.runtimes ?: "" + def sanitizer = config.sanitizer ?: "" + def assertions = config.assertions ?: false + def timeout = config.timeout ?: 120 + def buildTarget = config.build_target ?: "" + def noinstall = config.noinstall ?: false + def extraCmakeFlags = config.cmake_flags ?: [] + def stage1Mode = config.stage1 ?: false + def extraEnvVars = config.env_vars ?: [:] + def testCommand = config.test_command ?: "cmake" + def testTargets = config.test_targets ?: [] + + // Build environment variables map + def envVars = [ + "PATH": "\$PATH:/usr/bin:/usr/local/bin", + "MACOSX_DEPLOYMENT_TARGET": stage1Mode ? "13.6" : null + ] + + // Add custom environment variables + extraEnvVars.each { key, value -> + envVars[key] = value + } + + // Filter out null values + envVars = envVars.findAll { k, v -> v != null } + + def envList = envVars.collect { k, v -> "${k}=${v}" } + + withEnv(envList) { + timeout(timeout) { + withCredentials([string(credentialsId: 's3_resource_bucket', variable: 'S3_BUCKET')]) { + // Build the command dynamically + def buildCmd = buildMonorepoBuildCommand(config) + + sh """ + set -u + ${stage1Mode ? 'rm -rf build.properties' : ''} + source ./venv/bin/activate + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=\$(git describe --match "first_commit") + export GIT_DISTANCE=\$(echo \${git_desc} | cut -f 2 -d "-") + + sha=\$(echo \${git_desc} | cut -f 3 -d "-") + export GIT_SHA=\${sha:1} + + ${stage1Mode ? 'export LLVM_REV=$(git show -q | grep "llvm-svn:" | cut -f2 -d":" | tr -d " ")' : ''} + + cd - + + ${stage1Mode ? 'echo "GIT_DISTANCE=\$GIT_DISTANCE" > build.properties' : ''} + ${stage1Mode ? 'echo "GIT_SHA=\$GIT_SHA" >> build.properties' : ''} + ${stage1Mode ? 'echo "ARTIFACT=\$JOB_NAME/clang-d\$GIT_DISTANCE-g\$GIT_SHA-t\$BUILD_ID-b\$BUILD_NUMBER.tar.gz" >> build.properties' : ''} + + ${stage1Mode ? 'rm -rf clang-build clang-install *.tar.gz' : ''} + ${buildCmd} + """ + } + } + } + } + + static def buildMonorepoBuildCommand(config) { + def testCommand = config.test_command ?: "cmake" + def projects = config.projects ?: "clang;clang-tools-extra;compiler-rt" + def runtimes = config.runtimes ?: "" + def cmakeType = config.cmake_type ?: "RelWithDebInfo" + def assertions = config.assertions ?: false + def timeout = config.timeout ?: 120 + def buildTarget = config.build_target ?: "" + def noinstall = config.noinstall ?: false + def thinlto = config.thinlto ?: false + def sanitizer = config.sanitizer ?: "" + def extraCmakeFlags = config.cmake_flags ?: [] + + // Start building command + def cmd = "python llvm-zorg/zorg/jenkins/monorepo_build.py ${testCommand} build" + + // Add cmake type if not default + if (cmakeType != "default") { + cmd += " --cmake-type=${cmakeType}" + } + + // Add projects + cmd += " --projects=\"${projects}\"" + + // Add runtimes if specified + if (runtimes) { + cmd += " --runtimes=\"${runtimes}\"" + } + + // Add assertions flag + if (assertions) { + cmd += " --assertions" + } + + // Add timeout if different from default + if (timeout != 2400) { + cmd += " --timeout=${timeout}" + } + + // Add build target if specified + if (buildTarget) { + cmd += " --cmake-build-target=${buildTarget}" + } + + // Add noinstall flag + if (noinstall) { + cmd += " --noinstall" + } + + // Build cmake flags + def cmakeFlags = [] + cmakeFlags.add("-DPython3_EXECUTABLE=\$(which python)") + + if (thinlto) { + cmakeFlags.add("-DLLVM_ENABLE_LTO=Thin") + } + + if (sanitizer) { + cmakeFlags.add("-DLLVM_USE_SANITIZER=${sanitizer}") + } + + // Add DYLD_LIBRARY_PATH for TSan + if (sanitizer == "Thread") { + cmakeFlags.add("-DDYLD_LIBRARY_PATH=\$DYLD_LIBRARY_PATH") + } + + // Add extra cmake flags from config + cmakeFlags.addAll(extraCmakeFlags) + + // Add all cmake flags to command + cmakeFlags.each { flag -> + cmd += " --cmake-flag=\"${flag}\"" + } + + return cmd + } + + static def testStage(config = [:]) { + def testCommand = config.test_command ?: "cmake" + def testType = config.test_type ?: "testlong" // testlong vs test + def testTargets = config.test_targets ?: [] + def timeout = config.test_timeout ?: 420 + def extraEnvVars = config.env_vars ?: [:] + + // Build environment variables map + def envVars = [ + "PATH": "\$PATH:/usr/bin:/usr/local/bin" + ] + + // Add custom environment variables (like ASAN_SYMBOLIZER_PATH) + extraEnvVars.each { key, value -> + envVars[key] = value + } + + def envList = envVars.collect { k, v -> "${k}=${v}" } + + withEnv(envList) { + timeout(timeout) { + // Build test command dynamically + def cmd = "python llvm-zorg/zorg/jenkins/monorepo_build.py ${testCommand} ${testType}" + + // Add specific test targets if provided + testTargets.each { target -> + cmd += " --cmake-test-target=${target}" + } + + sh """ + set -u + source ./venv/bin/activate + + rm -rf clang-build/testresults.xunit.xml + + ${cmd} + """ + } + } + } + } + + static def cleanupStage() { + sh "rm -rf clang-build clang-install host-compiler *.tar.gz" + } + + static def createTemplatedPipeline(config) { + def jobName = config.name + def jobTemplate = config.job_template ?: 'clang-stage2-Rthinlto' + def enableBisectionTrigger = config.enable_bisection_trigger ?: false + def bisectJobName = config.bisect_job_name ?: 'build-bisect' + def descriptionPrefix = config.description_prefix ?: "" + + def clangBuilder = evaluate readTrusted('zorg/jenkins/lib/builders/ClangBuilder.groovy') + + return { + pipeline { + options { + disableConcurrentBuilds() + } + + parameters { + string(name: 'LABEL', defaultValue: params.LABEL ?: 'macos-x86_64', description: 'Node label to run on') + string(name: 'GIT_SHA', defaultValue: params.GIT_REVISION ?: '*/main', description: 'Git commit to build.') + string(name: 'ARTIFACT', defaultValue: params.ARTIFACT ?: 'llvm.org/clang-stage1-RA/latest', description: 'Clang artifact to use') + booleanParam(name: 'IS_BISECT_JOB', defaultValue: params.IS_BISECT_JOB ?: false, description: 'Whether this is a bisection job') + string(name: 'BISECT_GOOD', defaultValue: params.BISECT_GOOD ?: '', description: 'Good commit for bisection') + string(name: 'BISECT_BAD', defaultValue: params.BISECT_BAD ?: '', description: 'Bad commit for bisection') + } + + agent { + node { + label params.LABEL + } + } + + stages { + stage('Setup Build Description') { + steps { + script { + // Set build description based on context + def buildType = params.IS_BISECT_JOB ? "🔍 BISECTION TEST" : "🔧 NORMAL BUILD" + def commitInfo = params.GIT_SHA.take(8) + + if (params.IS_BISECT_JOB && params.BISECT_GOOD && params.BISECT_BAD) { + def goodShort = params.BISECT_GOOD.take(8) + def badShort = params.BISECT_BAD.take(8) + currentBuild.description = "${buildType}: Testing ${commitInfo} (${goodShort}..${badShort})" + } else { + currentBuild.description = "${buildType}: ${commitInfo}" + } + + if (descriptionPrefix) { + currentBuild.description = "${descriptionPrefix}: ${currentBuild.description}" + } + + echo "Build Type: ${buildType}" + echo "Job Template: ${jobTemplate}" + if (params.IS_BISECT_JOB) { + echo "This is a bisection test run - results will be used by build-bisect job" + } else { + echo "This is a normal CI build" + } + } + } + } + stage('Checkout') { + steps { + script { + clangBuilder.checkoutStage() + } + } + } + + stage('Setup Venv') { + steps { + script { + clangBuilder.setupVenvStage() + } + } + } + + stage('Fetch Artifact') { + when { + expression { + // Load template to check if this is a stage2+ build + def template = evaluate readTrusted("zorg/jenkins/lib/templates/${jobTemplate}.groovy") + def buildConfig = template.getDefaultBuildConfig() + def stage = buildConfig.stage ?: 2 // Default to stage2 if not specified + return stage >= 2 + } + } + steps { + script { + clangBuilder.fetchArtifactStage() + } + } + } + + stage('Build') { + steps { + script { + // Load the shared template + def template = evaluate readTrusted("zorg/jenkins/lib/templates/${jobTemplate}.groovy") + def buildConfig = template.getDefaultBuildConfig() + + clangBuilder.buildStage(buildConfig) + } + } + } + + stage('Test') { + steps { + script { + // Load the shared template + def template = evaluate readTrusted("zorg/jenkins/lib/templates/${jobTemplate}.groovy") + def testConfig = template.getDefaultTestConfig() + + clangBuilder.testStage(testConfig) + } + } + post { + always { + script { + junit "clang-build/**/testresults.xunit.xml" + } + } + } + } + } + + post { + always { + script { + clangBuilder.cleanupStage() + } + } + failure { + script { + // Only trigger bisection if enabled and this is not already a bisection job + if (enableBisectionTrigger && !params.IS_BISECT_JOB && shouldTriggerBisection()) { + triggerBisection(jobName, bisectJobName, jobTemplate) + } + } + } + } + } + } + } + + static def shouldTriggerBisection() { + // Check if this is a new failure by looking at previous build result + def previousBuild = currentBuild.previousBuild + if (previousBuild == null) { + return false // First build, can't bisect + } + + // Only bisect if previous build was successful (new failure) + return previousBuild.result == 'SUCCESS' + } + + static def triggerBisection(currentJobName, bisectJobName, jobTemplate) { + // Get the commit range for bisection + def currentCommit = env.GIT_COMMIT + def goodCommit = getPreviousGoodCommit() + + if (goodCommit) { + echo "Triggering bisection: ${goodCommit}...${currentCommit}" + + // Launch the bisection orchestrator with template configuration + build job: bisectJobName, + parameters: [ + string(name: 'BISECT_GOOD', value: goodCommit), + string(name: 'BISECT_BAD', value: currentCommit), + string(name: 'JOB_TEMPLATE', value: jobTemplate), + string(name: 'BUILD_CONFIG', value: '{}'), // Use template defaults + string(name: 'ARTIFACT', value: params.ARTIFACT) + ], + wait: false + } else { + echo "Could not determine good commit for bisection" + } + } + + static def getPreviousGoodCommit() { + // Walk back through builds to find the last successful one + def build = currentBuild.previousBuild + while (build != null) { + if (build.result == 'SUCCESS') { + // Extract commit from the successful build + def buildEnv = build.getBuildVariables() + return buildEnv.GIT_COMMIT + } + build = build.previousBuild + } + return null + } +} + +return this \ No newline at end of file diff --git a/zorg/jenkins/lib/templates/clang-stage2-Rthinlto.groovy b/zorg/jenkins/lib/templates/clang-stage2-Rthinlto.groovy new file mode 100644 index 000000000..3bed98c4c --- /dev/null +++ b/zorg/jenkins/lib/templates/clang-stage2-Rthinlto.groovy @@ -0,0 +1,41 @@ +#!/usr/bin/env groovy + +// Template configuration for clang ThinLTO jobs (Release + ThinLTO) +class ClangThinLTOTemplate { + + static def getDefaultBuildConfig(userConfig = [:]) { + def defaults = [ + thinlto: true, + test_command: "clang", + projects: "clang;compiler-rt", + cmake_type: "Release" // R = Release + ] + + // User config overrides defaults + return defaults + userConfig + } + + static def getDefaultTestConfig(userConfig = [:]) { + def defaults = [ + test_command: "clang" + ] + + return defaults + userConfig + } + + static def getJobDescription() { + return "Clang ThinLTO build configuration (Release + ThinLTO)" + } + + // Template-specific job configuration + static def getJobConfig(jobName) { + return [ + name: jobName, + job_template: 'clang-stage2-Rthinlto', + enable_bisection_trigger: true, // All templated ThinLTO jobs enable bisection + bisect_job_name: 'build-bisect' + ] + } +} + +return ClangThinLTOTemplate \ No newline at end of file diff --git a/zorg/jenkins/lib/utils/BisectionUtils.groovy b/zorg/jenkins/lib/utils/BisectionUtils.groovy new file mode 100644 index 000000000..c558703b8 --- /dev/null +++ b/zorg/jenkins/lib/utils/BisectionUtils.groovy @@ -0,0 +1,200 @@ +#!/usr/bin/env groovy + +class BisectionUtils { + + // Static list to track bisection steps for reproduction + static def bisectionSteps = [] + static def bisectionStartTime = 0 + static def stepDurations = [] + + static def performBisectionWithRunner(goodCommit, badCommit, jobTemplate, buildConfig, artifact) { + // Initialize bisection tracking + bisectionSteps.clear() + bisectionSteps.add("git bisect start --first-parent") + bisectionSteps.add("git bisect bad ${badCommit}") + bisectionSteps.add("git bisect good ${goodCommit}") + + bisectionStartTime = System.currentTimeMillis() + stepDurations.clear() + + // Calculate and log estimated steps + def commits = getCommitRange(goodCommit, badCommit) + def estimatedSteps = Math.ceil(Math.log(commits.size()) / Math.log(2)) + echo "Starting bisection: ${commits.size()} commits to test, estimated ${estimatedSteps} steps" + + return performBisectionRecursive(goodCommit, badCommit, jobTemplate, buildConfig, artifact) + } + + static def performBisectionRecursive(goodCommit, badCommit, jobTemplate, buildConfig, artifact) { + def commits = getCommitRange(goodCommit, badCommit) + + if (commits.size() <= 2) { + echo "Bisection complete: failing commit is ${badCommit}" + return badCommit + } + + def midpoint = commits[commits.size() / 2] + + // Calculate progress and ETA + def remainingCommits = commits.size() + def remainingSteps = Math.ceil(Math.log(remainingCommits) / Math.log(2)) + def avgStepDuration = stepDurations.size() > 0 ? stepDurations.sum() / stepDurations.size() : 0 + + def etaText = "unknown" + if (avgStepDuration > 0) { + def etaMillis = remainingSteps * avgStepDuration + def etaDays = Math.floor(etaMillis / 86400000) // 24 * 60 * 60 * 1000 + def etaHours = Math.floor((etaMillis % 86400000) / 3600000) // 60 * 60 * 1000 + + if (etaDays > 0) { + etaText = "${etaDays}d ${etaHours}h" + } else { + etaText = "${etaHours}h" + } + } + + echo "Bisecting: testing commit ${midpoint} (${remainingCommits} commits remaining, ~${remainingSteps} steps left, ETA: ${etaText})" + + // Record step start time + def stepStartTime = System.currentTimeMillis() + + // Test the midpoint commit using the build-bisect-run job + def testResult = testCommitWithRunner(midpoint, jobTemplate, buildConfig, artifact, goodCommit, badCommit) + + // Record step duration + def stepDuration = System.currentTimeMillis() - stepStartTime + stepDurations.add(stepDuration) + + // Format step duration for logging + def stepHours = Math.floor(stepDuration / 3600000) + def stepMinutes = Math.ceil((stepDuration % 3600000) / 60000) + echo "Step completed in ${stepHours}h ${stepMinutes}m" + + if (testResult == 'SUCCESS') { + // Failure is in the second half + bisectionSteps.add("git bisect good ${midpoint}") + return performBisectionRecursive(midpoint, badCommit, jobTemplate, buildConfig, artifact) + } else { + // Failure is in the first half + bisectionSteps.add("git bisect bad ${midpoint}") + return performBisectionRecursive(goodCommit, midpoint, jobTemplate, buildConfig, artifact) + } + } + + static def testCommitWithRunner(commit, jobTemplate, buildConfig, artifact, goodCommit, badCommit) { + echo "Testing commit ${commit} using job template ${jobTemplate}" + + def result = build job: 'build-bisect-run', + parameters: [ + string(name: 'GIT_SHA', value: commit), + string(name: 'JOB_TEMPLATE', value: jobTemplate), + string(name: 'BUILD_CONFIG', value: buildConfig), + string(name: 'ARTIFACT', value: artifact), + string(name: 'BISECT_GOOD', value: goodCommit), + string(name: 'BISECT_BAD', value: badCommit) + ], + propagate: false + + echo "Test result for ${commit}: ${result.result}" + return result.result + } + + static def getCommitRange(goodCommit, badCommit) { + def commits = sh( + script: "cd llvm-project && git rev-list --reverse ${goodCommit}..${badCommit}", + returnStdout: true + ).trim().split('\n') + + // Add the boundary commits + return [goodCommit] + commits + [badCommit] + } + + static def reportBisectionResult(failingCommit, originalJobName) { + def commitInfo = getCommitInfo(failingCommit) + + // Add final bisect step + bisectionSteps.add("git bisect reset") + + // Calculate final timing statistics + def totalDuration = System.currentTimeMillis() - bisectionStartTime + def totalDays = Math.floor(totalDuration / 86400000) + def totalHours = Math.floor((totalDuration % 86400000) / 3600000) + def totalMinutes = Math.ceil((totalDuration % 3600000) / 60000) + + def avgStepDuration = stepDurations.size() > 0 ? stepDurations.sum() / stepDurations.size() : 0 + def avgStepHours = Math.floor(avgStepDuration / 3600000) + def avgStepMinutes = Math.ceil((avgStepDuration % 3600000) / 60000) + + // Format total time + def totalTimeText = "" + if (totalDays > 0) { + totalTimeText = "${totalDays}d ${totalHours}h ${totalMinutes}m" + } else if (totalHours > 0) { + totalTimeText = "${totalHours}h ${totalMinutes}m" + } else { + totalTimeText = "${totalMinutes}m" + } + + // Format average step time + def avgStepText = "${avgStepHours}h ${avgStepMinutes}m" + + // Format reproduction steps + def reproductionSteps = bisectionSteps.join('\n') + + def report = """ +=== BISECTION COMPLETE === +Original Job: ${originalJobName} +Failing commit: ${failingCommit} +Author: ${commitInfo.author} +Date: ${commitInfo.date} +Message: ${commitInfo.message} + +Bisection Statistics: +- Total steps: ${stepDurations.size()} +- Total time: ${totalTimeText} +- Average step time: ${avgStepText} + +This commit appears to be the first one that introduced the failure. + +To reproduce this bisection locally: +1. Clone the repository and navigate to llvm-project/ +2. Run the following git bisect commands in sequence: + +${reproductionSteps} + +3. At each bisect step, test the commit using your build configuration +4. Mark commits as 'git bisect good' or 'git bisect bad' based on build results +5. The bisection will converge on commit ${failingCommit} + +To reproduce the specific failure: +1. Check out commit ${failingCommit} +2. Run the ${originalJobName} job configuration +3. The failure should reproduce consistently + +=========================== + """.trim() + + echo report + + // Write the report to the workspace for archival + writeFile file: 'bisection-result.txt', text: report + archiveArtifacts artifacts: 'bisection-result.txt', allowEmptyArchive: false + + return failingCommit + } + + static def getCommitInfo(commit) { + def authorInfo = sh( + script: "cd llvm-project && git show -s --format='%an|%ad|%s' ${commit}", + returnStdout: true + ).trim().split('\\|') + + return [ + author: authorInfo[0], + date: authorInfo[1], + message: authorInfo[2] + ] + } +} + +return this \ No newline at end of file