From fb63a3949fa4e4ebbb628491fadb8b25542e1d61 Mon Sep 17 00:00:00 2001 From: Mick Semb Wever Date: Fri, 10 Jul 2026 20:16:47 +0200 Subject: [PATCH] jenkins shouldn't retry on fail-fast exceptions To jenkins k8s add ephemeral-storage limits to agents and log peak disk usage Agents were being evicted at the node level due to missing ephemeral-storage requests/limits and no sizeLimit on the /var/lib/docker emptyDir. Also extract inline sh blocks to functions for readability --- .build/run-ci | 15 ++++ .jenkins/Jenkinsfile | 106 ++++++++++++++++++--------- .jenkins/k8s/jenkins-deployment.yaml | 24 +++++- 3 files changed, 111 insertions(+), 34 deletions(-) diff --git a/.build/run-ci b/.build/run-ci index 2cb0f6cb61ad..b62f69fc953c 100755 --- a/.build/run-ci +++ b/.build/run-ci @@ -38,6 +38,7 @@ import gzip import itertools import os import shutil +import socket import subprocess import sys import tarfile @@ -275,6 +276,19 @@ def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_ sys.exit(1) +def wait_for_jenkins_http(ip: str, timeout: int = 300): + host, port = (ip.rsplit(":", 1)[0], int(ip.rsplit(":", 1)[1])) if ":" in ip else (ip, 80) + spin_while(f"Waiting for Jenkins HTTP at {host}:{port}… ", lambda: _tcp_connect_ok(host, port)) + + +def _tcp_connect_ok(host: str, port: int) -> bool: + try: + with socket.create_connection((host, port), timeout=2): + return True + except OSError: + return False + + def get_jenkins(k8s_client: client.CoreV1Api, args, kube_ns: str) -> Tuple[str, jenkins.Jenkins]: """Authenticates to Jenkins and returns the Jenkins ip and server objects.""" @@ -828,6 +842,7 @@ def main(): (ip, server) = get_jenkins(k8s_client, args, DEFAULT_KUBE_NS) if args.setup or args.only_setup: + wait_for_jenkins_http(ip) ensure_cassandra_job_parameters_visible(server) if args.only_setup: return diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index 3809eecafd25..83532a06baf8 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -352,7 +352,7 @@ def build(command, cell) { ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") { try { fetchSource(cell.step, cell.arch, cell.jdk) - sh """ + sh label: "checking Jenkinsfile validity", script: """ test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; } grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; } """ @@ -370,7 +370,7 @@ def build(command, cell) { } if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } if ("jar" == cell.step) { - stash name: "${cell.arch}_${cell.jdk}" + _stash(cell) } dir("build") { copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/") @@ -438,8 +438,8 @@ def test(command, cell) { def descriptions = [] for (def cause in exc.getCauses()) { echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}" - if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) { - throw exc // user explicitly aborted — do not retry + if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption') || cause.getClass().getName().contains('ParallelStep$FailFastCause')) { + throw exc // user abort or fail-fast — do not retry } descriptions.add(cause.getShortDescription()) } @@ -453,24 +453,12 @@ def test(command, cell) { } } dir("build") { - sh """ - mkdir -p test/output/${cell.step} - find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';' - find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';' - find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';' - """ + organiseTestResultFiles(cell, cell_suffix) if (!cell.step.startsWith("microbench")) { junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']] } - // check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes - sh """ - echo "docker memory/oomkiller debug:" - cat /sys/fs/cgroup/docker/memory.events || true - """ - sh """ - find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f - echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l - """ + debugOomKiller() + compressTestResultFiles() archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) } @@ -517,7 +505,7 @@ def fetchDockerImages(dockerfiles) { // prefetch, from apache jfrog, reduces risking dockerhub pull rate limits // also prefetch alpine:latest as its used as a utility in the scripts def dockerfilesVar = dockerfiles.join(' ') - sh """#!/bin/bash + sh label: "fetching docker images...", script: """#!/bin/bash for dockerfile in ${dockerfilesVar} ; do image_tag="\$(md5sum .build/docker/\${dockerfile}.docker | cut -d' ' -f1)" image_name="apache/cassandra-\${dockerfile}:\${image_tag}" @@ -527,6 +515,13 @@ def fetchDockerImages(dockerfiles) { done docker pull -q apache.jfrog.io/cassan-docker/alpine:3.19.1 & wait + # debug how much space the images have taken (default node disk-size is 100G) + { set +x; } 2>/dev/null + free_gib=\$(df --output=avail -B 1073741824 / | tail -1 | tr -d ' ') + echo "docker images total: \$(docker system df --format '{{.Size}}' | head -1) (free: \${free_gib}Gi)" + if [ "\${free_gib}" -le 10 ] ; then + echo "WARNING: only \${free_gib}Gi free after pulling images — review volume-0 emptyDir sizeLimit in .jenkins/k8s/jenkins-deployment.yaml" + fi """ } @@ -564,15 +559,15 @@ def copyToNightlies(sourceFiles, remoteDirectory='') { def cleanAgent(job_name) { // get any public IP which is more helpful correlating back to the cloud instance sh script: 'hostname; curl -sm 10 ifconfig.me', returnStatus: true + def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/" if (isCanonical()) { - def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/" cleanAgentDocker(job_name, agentScriptsUrl) - logAgentInfo(job_name, agentScriptsUrl) } + logAgentInfo(job_name, agentScriptsUrl) cleanWs() if (isCanonical()) { // in the workspace prune any abandoned or uncleaned builds (CASSANDRA-20436) - sh """#!/bin/bash + sh label: "prune abandoned and uncleaned workspace builds files...", script: """#!/bin/bash set +e find /home/jenkins/jenkins-*/workspace/ -mindepth 2 -maxdepth 2 -type d -regextype posix-extended -regex '.*/[0-9]+' -mtime +31 -print -exec rm -rf {} + """ @@ -582,8 +577,7 @@ def cleanAgent(job_name) { def cleanAgentDocker(job_name, agentScriptsUrl) { // we don't expect any build to have been running for longer than maxBuildHours def maxBuildHours = 12 - echo "Pruning docker for '${job_name}' on ${NODE_NAME}…" ; - sh """#!/bin/bash + sh label: "Pruning docker for '${job_name}' on ${NODE_NAME}...", script: """#!/bin/bash set +e wget -q ${agentScriptsUrl}/docker_image_pruner.py wget -q ${agentScriptsUrl}/docker_agent_cleaner.sh @@ -592,12 +586,58 @@ def cleanAgentDocker(job_name, agentScriptsUrl) { } def logAgentInfo(job_name, agentScriptsUrl) { - sh """#!/bin/bash - set +e -o pipefail - wget -q ${agentScriptsUrl}/agent_report.sh - bash -x agent_report.sh | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt - """ - copyToNightlies("*-disk-usage-stats.txt", "cassandra/ci-cassandra.apache.org/agents/${NODE_NAME}/disk-usage/") + // post-run remaining build/ and docker usage. used to validate volume-0 sizeLimit + sh label: "log build usage...", script: """ + { set +x; } 2>/dev/null + du -sh ${WORKSPACE}/build/m2 ${WORKSPACE}/build/tmp ${WORKSPACE}/build/test ${WORKSPACE}/build 2>/dev/null || true + df -h / || true + """ + if (isCanonical()) { + sh label: "running agent_report.sh for disk usage stats (and more)...", script: """#!/bin/bash + set +e -o pipefail + wget -q ${agentScriptsUrl}/agent_report.sh + bash -x agent_report.sh | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt + """ + copyToNightlies("*-disk-usage-stats.txt", "cassandra/ci-cassandra.apache.org/agents/${NODE_NAME}/disk-usage/") + } +} + +def _stash(cell) { + sh label: "check stash size...", script: """ + { set +x; } 2>/dev/null + free_gib=\$(df --output=avail -B 1073741824 / | tail -1 | tr -d ' ') + stash_gb=\$(du -sb ${WORKSPACE} | awk '{printf "%.1f", \$1/1024/1024/1024}') + echo "stash size: \${stash_gb}G (free: \${free_gib}Gi)" + if [ "\${free_gib}" -le 10 ] ; then + echo "WARNING: only \${free_gib}Gi free after building stash (\${stash_gb}G) — review jnlp resourceRequestEphemeralStorage in .jenkins/k8s/jenkins-deployment.yaml" + fi + """ + stash name: "${cell.arch}_${cell.jdk}" +} + +def organiseTestResultFiles(cell, cell_suffix) { + sh label: "organise test result files...", script: """ + mkdir -p test/output/${cell.step} + find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';' + find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';' + find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';' + """ +} + +def debugOomKiller() { + // check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes + sh label: "checking for oom kills...", script: """ + # docker memory/oomkiller debug: + cat /sys/fs/cgroup/docker/memory.events || true + """ +} + +def compressTestResultFiles() { + sh label: "compress test result files...", script: """ + { set +x; } 2>/dev/null + find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f + echo "\$(find test/output -type f -name "*.xml.xz" | wc -l) test result files compressed" + """ } ///////////////////////////////////////// @@ -614,7 +654,7 @@ def generateTestReports() { def script_vars = "#!/bin/bash -x \n " if (isCanonical()) { // copyArtifacts takes >4hrs, hack with manual download - sh """${script_vars} + sh label: "manual download (instead of copyArtifacts)...", script: """${script_vars} ( mkdir -p build/test wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip unzip -x -d build/test -q output.zip ) ${teeSuffix} @@ -627,7 +667,7 @@ def generateTestReports() { // merge splits for each target's test report, other axes are kept separate // TODO parallelised for loop // TODO results_details.tar.xz needs to include all logs for failed tests - sh """${script_vars} ( + sh label: "merging splits test reports...", script: """${script_vars} ( echo "test result files to decompress"; find build/test/output -type f -name "*.xml.xz" | wc -l find build/test/output -type f -name "*.xml.xz" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f --decompress diff --git a/.jenkins/k8s/jenkins-deployment.yaml b/.jenkins/k8s/jenkins-deployment.yaml index 98290e601158..2170dba45821 100644 --- a/.jenkins/k8s/jenkins-deployment.yaml +++ b/.jenkins/k8s/jenkins-deployment.yaml @@ -69,6 +69,7 @@ controller: - "staticMethod org.codehaus.groovy.runtime.DefaultGroovyMethods inspect java.lang.Object" - "staticMethod org.codehaus.groovy.runtime.DefaultGroovyMethods max java.util.Collection" - "staticMethod org.codehaus.groovy.runtime.DefaultGroovyMethods putAt java.util.List java.util.List java.lang.Object" + - "field hudson.plugins.git.GitSCMBackwardCompatibility branch" - "method org.jenkinsci.plugins.workflow.steps.FlowInterruptedException getCauses" JCasC: configScripts: @@ -213,7 +214,7 @@ agent: key: "DOCKER_IPTABLES_LEGACY" value: "1" image: docker:dind - args: "--default-address-pool base=192.168.96.0/20,size=24" # overwrite docker subnet in case of overlapping + args: "--default-address-pool base=192.168.96.0/20,size=24" # overwrite docker subnet in case of overlapping livenessProbe: failureThreshold: '0' initialDelaySeconds: '0' @@ -247,6 +248,13 @@ agent: values: - "true" topologyKey: kubernetes.io/hostname + volumes: + # volume-0 is /var/lib/docker. sizeLimit triggers a clean pod-level eviction with a clear message + # before the node's own eviction threshold fires. ubuntu-test image is ~35GB; 45Gi leaves ~10Gi headroom (based on a node's default --disk-size of 100G) + # if the image grows and this limit is hit, fetchDockerImages in Jenkinsfile should warn first. + - name: volume-0 + emptyDir: + sizeLimit: 45Gi agent-dind-medium: | - name: agent-dind-medium label: agent-dind cassandra-medium cassandra-amd64-medium @@ -337,6 +345,13 @@ agent: values: - "true" topologyKey: kubernetes.io/hostname + volumes: + # volume-0 is /var/lib/docker. sizeLimit triggers a clean pod-level eviction with a clear message + # before the node's own eviction threshold fires. ubuntu-test image is ~35GB; 45Gi leaves ~10Gi headroom (based on a node's default --disk-size of 100G) + # if the image grows and this limit is hit, fetchDockerImages in Jenkinsfile should warn first. + - name: volume-0 + emptyDir: + sizeLimit: 45Gi agent-dind-large: | - name: agent-dind-large label: agent-dind cassandra-large cassandra-amd64-large cassandra-amd64-large-dedicated @@ -427,5 +442,12 @@ agent: values: - "true" topologyKey: kubernetes.io/hostname + volumes: + # volume-0 is /var/lib/docker. sizeLimit triggers a clean pod-level eviction with a clear message + # before the node's own eviction threshold fires. ubuntu-test image is ~35GB; 45Gi leaves ~10Gi headroom (based on a node's default --disk-size of 100G) + # if the image grows and this limit is hit, fetchDockerImages in Jenkinsfile should warn first. + - name: volume-0 + emptyDir: + sizeLimit: 45Gi