openjdk_tests

#!groovy

def PLATFORM_MAP = [
    'arm_linux' : [
        'SPEC' : 'linux_arm',
        'LABEL' : 'ci.role.test&&sw.os.linux&&hw.arch.aarch32',
    ],
    'aarch64_mac' : [
        'SPEC' : 'osx_aarch64',
        'LABEL' : 'ci.role.test&&hw.arch.aarch64&&(sw.os.osx||sw.os.mac)',
    ],
    'aarch64_linux' : [
        'SPEC' : 'linux_aarch64',
        'LABEL' : 'ci.role.test&&sw.os.linux&&hw.arch.aarch64',
    ],
    'aarch64_windows' : [
        'SPEC' : 'windows_aarch64',
        'LABEL' : 'ci.role.test&&sw.os.windows&&hw.arch.aarch64',
    ],
    'aarch64_alpine-linux' : [
        'SPEC' : 'alpine-linux_aarch64',
        'LABEL' : 'ci.role.test&&hw.arch.aarch64&&sw.os.alpine-linux',
    ],
    'ppc32_aix' : [
        'SPEC' : 'aix_ppc',
        'LABEL' : 'ci.role.test&&hw.arch.ppc64&&sw.os.aix',
    ],
    'ppc32_linux' : [
        'SPEC' : 'linux_ppc',
        'LABEL' : 'ci.role.test&&hw.arch.ppc64&&sw.os.linux',
    ],
    'ppc64_aix' : [
        'SPEC' : 'aix_ppc-64',
        'LABEL' : 'ci.role.test&&hw.arch.ppc64&&sw.os.aix',
    ],
    'ppc64_linux' : [
        'SPEC' : 'linux_ppc-64',
        'LABEL' : 'ci.role.test&&hw.arch.ppc64&&sw.os.linux',
    ],
    'ppc64le_linux' : [
        'SPEC' : 'linux_ppc-64_le',
        'LABEL' : 'ci.role.test&&hw.arch.ppc64le&&sw.os.linux',
        'DynamicAgents' : ['fyre']
    ],
    'riscv64_linux' : [
        'SPEC' : 'linux_riscv64',
        'LABEL' : 'ci.role.test&&sw.os.linux&&hw.arch.riscv',
        'DockerAgents' : ['default']
    ],
    'riscv64_linux_xl' : [
        'SPEC' : 'linux_riscv64',
        'LABEL' : 'ci.role.test&&sw.os.linux&&hw.arch.riscv&&hw.bits.64',
    ],
    's390_linux' : [
        'SPEC' : 'linux_390',
        'LABEL' : 'ci.role.test&&hw.arch.s390x&&sw.os.linux&&hw.bits.32',
    ],
    's390x_linux_zt' : [
        'SPEC' : 'linux_390-64_zt',
        'LABEL' : 'ci.role.test&&hw.arch.znext&&sw.os.linux',
    ],
    's390x_linux' : [
        'SPEC' : 'linux_390-64',
        'LABEL' : 'ci.role.test&&hw.arch.s390x&&sw.os.linux',
    ],
    's390x_zos' : [
        'SPEC' : 'zos_390-64',
        'LABEL' : 'ci.role.test&&hw.arch.s390x&&sw.os.zos',
    ],
    's390x_zos_xl' : [
        'SPEC' : 'zos_390-64',
        'LABEL' : 'ci.role.test&&hw.arch.s390x&&sw.os.zos',
    ],
    's390x_zos_zt' : [
        'SPEC' : 'zos_390-64_zt',
        'LABEL' : 'ci.role.test&&hw.arch.znext&&sw.os.zos',
    ],
    's390_zos_zt' : [
        'SPEC' : 'zos_390_zt',
        'LABEL' : 'ci.role.test&&hw.arch.znext&&sw.os.zos',
    ],
    's390_zos' : [
        'SPEC' : 'zos_390',
        'LABEL' : 'ci.role.test&&hw.arch.s390x&&sw.os.zos',
    ],
    'sparcv9_solaris' : [
        'SPEC' : 'sunos_sparcv9-64',
        'LABEL' : 'ci.role.test&&hw.arch.sparcv9&&sw.os.sunos',
    ],
    'x86-64_solaris' : [
        'SPEC' : 'sunos_x86-64',
        'LABEL' : 'ci.role.test&&hw.arch.x86&&sw.os.sunos',
    ],
    'x86-64_alpine-linux' : [
        'SPEC' : 'alpine-linux_x86-64',
        'LABEL' : 'ci.role.test&&hw.arch.x86&&sw.os.alpine-linux',
    ],
    'x86-32_linux' : [
        'SPEC' : 'linux_x86',
        'LABEL' : 'ci.role.test&&hw.arch.x86&&sw.os.linux',
    ],
    'x86-32_windows' : [
        'SPEC' : 'win_x86',
        'LABEL' : 'ci.role.test&&hw.arch.x86&&sw.os.windows',
    ],
    'x86-64_linux' : [
        'SPEC' : 'linux_x86-64',
        'LABEL' : 'ci.role.test&&hw.arch.x86&&sw.os.linux',
        'DynamicAgents' : ['azure', 'fyre']
    ],
    'x86-64_mac' : [
        'SPEC' : 'osx_x86-64',
        'LABEL' : 'ci.role.test&&hw.arch.x86&&(sw.os.osx||sw.os.mac)',
    ],
    'x86-64_windows' : [
        'SPEC' : 'win_x86-64',
        'LABEL' : 'ci.role.test&&hw.arch.x86&&sw.os.windows',
    ],
]

if ( !params.TARGET ) {
    assert false : "Please provide TARGET value"
}
def PLATFORMS = params.PLATFORM.trim().split("\\s*,\\s*");
def JDK_VERSIONS = params.JDK_VERSION.trim().split("\\s*,\\s*");
def JDK_IMPLS = params.JDK_IMPL.trim().split("\\s*,\\s*");

// if multiple JDK_VERSION / JDK_IMPL / PLATFORM are provided, run test jobs in parallel
timestamps{
    if (JDK_VERSIONS.size() > 1 || JDK_IMPLS.size() > 1 || PLATFORMS.size() >1 || PLATFORMS.any { it.contains("all") }) {
        if (SDK_RESOURCE != 'nightly' && SDK_RESOURCE != 'releases') {
            assert false : "Multiple Grinders should run with SDK_RESOURCE=nightly or releases."
        } else {
            testJobs = [:]
            PLATFORMS.each { PLATFORM ->
                JDK_VERSIONS.each { JDK_VERSION ->
                    JDK_IMPLS.each { JDK_IMPL ->
                        def ACTUAL_PLATFORM = resolvePlatform(PLATFORM, JDK_VERSION)
                        def childParams = []
                        // loop through all the params and change the parameters if needed
                        params.each { param ->
                            if (param.key == "PLATFORM") {
                                childParams << string(name: param.key, value: ACTUAL_PLATFORM)
                            } else if (param.key == "JDK_VERSION") {
                                childParams << string(name: param.key, value: JDK_VERSION)
                            } else if (param.key == "JDK_IMPL") {
                                childParams << string(name: param.key, value: JDK_IMPL)
                            } else {
                                def value = param.value.toString()
                                if (value == "true" || value == "false") {
                                    childParams << booleanParam(name: param.key, value: value.toBoolean())
                                } else {
                                    childParams << string(name: param.key, value: value)
                                }
                            }
                        }
                        testJobs["openjdk${JDK_VERSION}_${JDK_IMPL}_${PLATFORM}"] = {
                            build job: JOB_NAME, parameters: childParams
                        }
                    }
                }
            }
            parallel testJobs
        }
    } else if (!JOB_NAME.contains("_imagePull") && !JOB_NAME.contains("_imageUpload") && params.TARGET == "dev.external" ) {
        // trigger xxx_imageUpload test job, then xxx_imagePull test job
        def commonLabel = "sw.tool.podman&&sw.tool.container.criu"
        if (params.LABEL_ADDITION) {
            commonLabel += "&&${params.LABEL_ADDITION}"
        }
        def imageUploadMap = [
            'x86-64_linux' : [
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22&&hw.arch.x86.broadwell"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22&&hw.arch.x86.amd"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.x86.broadwell"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.x86.amd"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.x86.skylake"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.9&&hw.arch.x86.amd"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.9&&hw.arch.x86.broadwell"]
                    ],
            'aarch64_linux' : [
                    	// no machine available ['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.20"],
                    	['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22&&hw.arch.aarch64.armv8"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.aarch64.armv8"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.9&&hw.arch.aarch64.armv8"]
                    ],
            's390x_linux' : [
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22&&hw.arch.s390x.z13"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22&&hw.arch.s390x.z14"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22&&hw.arch.s390x.z15"],
                        // no machine available ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.s390x.z13"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.s390x.z14"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.s390x.z15"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.9&&hw.arch.s390x.z14"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.9&&hw.arch.s390x.z15"]
                    ],
            'ppc64le_linux' : [
                        // no machine available ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.ppc64le.p9"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.ppc64le.p10"],
                        // no machine available ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.9&&hw.arch.ppc64le.p8"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.9&&hw.arch.ppc64le.p9"],
                        // no machine available ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.98&&hw.arch.ppc64le.p10"],
            ]
        ]
        def imagePullMap = [
            'x86-64_linux' : [
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.9"]
                    ],
            'aarch64_linux' : [
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8"],
                        ['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.9"]
                    ],
            's390x_linux' : [
                        ['LABEL_ADDITION' : "${commonLabel}&&hw.arch.s390x.z13"],
                        ['LABEL_ADDITION' : "${commonLabel}&&hw.arch.s390x.z14"],
                        ['LABEL_ADDITION' : "${commonLabel}&&hw.arch.s390x.z15"]
                    ],
            'ppc64le_linux' : [
                        // Temporarily exclude ubuntu 24 machines due to issue runtimes_backlog 1506
                        ['LABEL_ADDITION' : "${commonLabel}&&hw.arch.ppc64le.p10&&!sw.os.ubuntu.24"],
                        ['LABEL_ADDITION' : "${commonLabel}&&hw.arch.ppc64le.p9&&!sw.os.ubuntu.24"]
                    ]
        ]
        if (params.PLATFORM && imageUploadMap[params.PLATFORM] != null && imagePullMap[params.PLATFORM] != null) {
            def imageUploadJobName = "${JOB_NAME}_imageUpload"
            def imagePullJobName = "${JOB_NAME}_imagePull"
            generateJobViaAutoGen(imageUploadJobName)
            generateJobViaAutoGen(imagePullJobName)

            def imageUploadJobs = [:]
            def target = "testList TESTLIST=disabled.criu_pingPerf_testCreateRestoreImageAndPushToRegistry_ubi8,disabled.criu_pingPerf_testCreateRestoreImageAndPushToRegistry_ubi9,disabled.criu-portable-checkpoint_test,disabled.criu-ubi-portable-checkpoint_test"

            if (params.PLATFORM == "ppc64le_linux") {
                // exclude criu-portable-checkpoint_test on plinux due to github_ibm/runtimes/backlog/issues/1099
                target = target.minus("disabled.criu-portable-checkpoint_test,")
            }

            echo "Trigger ${imageUploadJobName} job ..."
            for (int i = 0; i < imageUploadMap[params.PLATFORM].size(); i++) {
                def labelAddition = imageUploadMap[params.PLATFORM][i].LABEL_ADDITION
                def newParams = ["LABEL_ADDITION" : labelAddition, "TARGET" : "${target}", "DOCKER_REGISTRY_DIR" : "${JOB_NAME}:${BUILD_NUMBER}"]
                def childParams = changeParam(params, newParams)
                imageUploadJobs["${imageUploadJobName}_${i}]"] = {
                    build job: imageUploadJobName, parameters: childParams
                }
            }
            parallel imageUploadJobs

            def imagePullJobs = [:]
            echo "Trigger ${imagePullJobName} job ..."
            for (int i = 0; i < imagePullMap[params.PLATFORM].size(); i++) {
                def labelAddition = imagePullMap[params.PLATFORM][i].LABEL_ADDITION
                def newParams = ["LABEL_ADDITION" : labelAddition, "DOCKER_REGISTRY_DIR" : "${JOB_NAME}:${BUILD_NUMBER}"]
                def childParams = changeParam(params, newParams)
                imagePullJobs["${imagePullJobName}_${i}]"] = {
                    build job: imagePullJobName, parameters: childParams
                }
            }
            parallel imagePullJobs
        } else {
             assert false : "Cannot find key PLATFORM: ${params.PLATFORM} in imageUploadMap: ${imageUploadMap} and/or imagePullMap: ${imagePullMap}."
        } 
    } else {
        if (PLATFORM_MAP.containsKey(params.PLATFORM)) {
            env.EXTRA_OPTIONS = params.EXTRA_OPTIONS ? params.EXTRA_OPTIONS : ""
            env.BUILD_LIST = params.BUILD_LIST ? params.BUILD_LIST : ""
            SPEC = PLATFORM_MAP[params.PLATFORM]["SPEC"]
            // Block dockerAgents on adoptium CI for riscv64 until we make it work
            // This will allow RISC-V test jobs to queue up and not hang
            if ( env.JENKINS_URL.contains("ci.adoptium.net")) {
                dockerAgents = null
            } else {
                dockerAgents = PLATFORM_MAP[params.PLATFORM]["DockerAgents"] ? PLATFORM_MAP[params.PLATFORM]["DockerAgents"] : []
            }
            dockerAgentLabel = ''
            if (params.LABEL) {
                LABEL = params.LABEL
            } else {
                LABEL = PLATFORM_MAP[params.PLATFORM]["LABEL"]
                    if (params.BUILD_LIST.contains("perf")) {
                        def perfLabel = LABEL.minus("ci.role.test&&").concat("&&ci.role.perf")
                        if (areNodesWithLabelOnline(perfLabel)) {
                        LABEL = perfLabel
                        }
                    }
                }

            // Temporarily support both FIPS and FIPS140_2, remove FIPS in the next step
            if (params.TEST_FLAG == "FIPS" || params.TEST_FLAG == "FIPS140_2") {
                LABEL = LABEL.minus("ci.role.test&&").concat("&&ci.role.test.fips")
                env.EXTRA_OPTIONS += " -Dsemeru.fips=true"
            } else if (params.TEST_FLAG.contains("FIPS140_3_")) {
                String[] tokens = TEST_FLAG.split('_')
                def opts = tokens[2];
                env.EXTRA_OPTIONS += " -Dsemeru.fips=true -Dsemeru.customprofile=${opts}"
            }

            // Temporarily exclude CentOS 7 machines for sanity.external pipeline due to docker issue
            if (JOB_NAME.contains ("sanity.external")) {
                LABEL += "&&!sw.os.cent.7"
            }

            // JDK23+ isn't supported on machines prior to Mac 11 https://github.com/eclipse-openj9/openj9/issues/19694
            // assume if JDK_VERSION is set to next, JDK_VERSION is 23+
            if (params.JDK_IMPL == "openj9" && PLATFORM == "x86-64_mac" && params.JDK_VERSION) {
                if (!params.JDK_VERSION.isInteger() || params.JDK_VERSION.toInteger() >= 23) {
                    LABEL += "&&!sw.os.mac.10"
                }
            }

            if (params.DOCKER_REQUIRED) {
                LABEL += "&&sw.tool.docker"
            }

            if (params.LABEL_ADDITION) {
                LABEL += "&&${params.LABEL_ADDITION}"
            }

            println "SPEC: ${SPEC}"
            println "LABEL: ${LABEL}"

            stage('Queue') {
                if (!areNodesWithLabelOnline(LABEL) && !dockerAgents) {
                    int ACTIVE_NODE_TIMEOUT = params.ACTIVE_NODE_TIMEOUT ? params.ACTIVE_NODE_TIMEOUT.toInteger() : 15
                    timeout(ACTIVE_NODE_TIMEOUT) {
                        // If there is available node before timeout
                        node(LABEL) {
                            echo "find the node with label as ${env.NODE_NAME}"
                        }
                    }
                } else {
                    // IF no nodes are idle we will check if there is supported virtual agent
                    // When Parallel the race condition could happen. Say the number of multiply jobs is larger than the available nodes the query's result may be delayed and wrong
                    // In this case jobs will be fooled to fall back to wait local busy nodes.
                    dynamicAgents = PLATFORM_MAP[params.PLATFORM]["DynamicAgents"] ? PLATFORM_MAP[params.PLATFORM]["DynamicAgents"] : []
                    println "dynamicAgents: ${dynamicAgents}"

                    if ((params.CLOUD_PROVIDER != null && params.CLOUD_PROVIDER in dynamicAgents) || dockerAgents) {
                        boolean isNodeIdle = false
                        node {
                            String[] onlineNodes = nodesByLabel(LABEL)
                            for (String onlineNode : onlineNodes) {
                                def currentNode = Jenkins.instance.getNode(onlineNode).getComputer()
                                if (!currentNode.isOffline()) {
                                    if (currentNode.countBusy() < currentNode.getNumExecutors()) {
                                        println "Found an idle node: ${onlineNode}. The program will not start dynamic vm."
                                        isNodeIdle = true
                                        break
                                    }
                                }
                            }
                        }
                        if (!isNodeIdle) {
                            // DynamicAgents and DockerAgents shouldn't contain the same label
                            if (params.CLOUD_PROVIDER != null && params.CLOUD_PROVIDER in dynamicAgents) {
                                LABEL = LABEL.minus("ci.role.test&&")
                                LABEL += '&&ci.agent.dynamic'
                                println "Cannot find any idle nodes. Starting dynamic vm, nodeLabel: '${LABEL}', dynamicLabel: '${params.CLOUD_PROVIDER}'"
                            } else if (params.CLOUD_PROVIDER != null && params.CLOUD_PROVIDER in dockerAgents) {
                                dockerAgentLabel = params.CLOUD_PROVIDER
                            } else if (dockerAgents.size() >= 1) {
                                dockerAgentLabel = dockerAgents[0]
                            }
                            if (!dockerAgentLabel.equals('')) {
                                if (dockerAgentLabel.equals('default') && SPEC.equals('linux_riscv64')) {
                                    LABEL = LABEL.minus("&&hw.arch.riscv")
                                    LABEL += '&&hw.arch.x86'
                                    if (!LABEL.contains('&&sw.tool.docker')) {
                                        LABEL += '&&sw.tool.docker'
                                    }
                                }
                                println "Cannot find any idle nodes. Starting run test on docker node, nodeLabel: '${LABEL}', dockerAgentLabel: '${dockerAgentLabel}'"
                            }
                        }
                    }
                }

                if (params.RELATED_NODES) {
                    if (areNodesWithLabelOnline(params.RELATED_NODES)) {
                        timeout(activity: true, time: 1, unit: 'HOURS'){
                            node(params.RELATED_NODES) {
                            echo "On RELATED_NODES: ${params.RELATED_NODES}"
                            env.AGENT_NODE = "${NODE_NAME}"
                            node(LABEL) {
                                echo "On main node"
                                runTest()
                                echo "Done with main node"
                                }
                                echo "Done with RELATED_NODES: ${params.RELATED_NODES}"
                            }
                        }
                    } else {
                        assert false : "Cannot find RELATED_NODES: ${params.RELATED_NODES}."
                    }
                } else {
                    node(LABEL) {
                        runTest()
                    }
                }
            }
            if (currentBuild.result != 'FAILURE') {
                env.SETUP_LABEL = params.SETUP_LABEL ?: "ci.role.test"
                echo "SETUP_LABEL: ${env.SETUP_LABEL}"
                def rerunTestJobParams = jenkinsfile.runParallelTests()
                jenkinsfile.triggerRerunJob(rerunTestJobParams)
            }
        } else {
            assert false : "Cannot find key PLATFORM: ${params.PLATFORM} in PLATFORM_MAP: ${PLATFORM_MAP}."
        }
    }
}

def runTest() {
    try {
        def retry_count = 0
        def sleep_time = 180

        if (params.PLATFORM.contains('zos')) {
            /* Ensure correct CC env */
            env._CC_CCMODE = '1'
            env._CXX_CCMODE = '1'
            env._C89_CCMODE = '1'

            def gitConfig = scm.getUserRemoteConfigs()[0]
            def SCM_GIT_REPO = gitConfig.getUrl()
            def SCM_GIT_BRANCH = scm.branches[0].name

            // SCM_GIT_REPO value only gets expanded in sh
            def SCM_GIT_REPO_VAL = sh(script: "echo ${SCM_GIT_REPO}", returnStdout: true).trim()
            SCM_GIT_REPO_VAL = SCM_GIT_REPO_VAL.replace("https://github.com/","git@github.com:")

            retry(5) {
                if (retry_count > 0) {
                    sleep(sleep_time)
                }
                retry_count++
                timeout(time: 1, unit: 'HOURS') {
                    forceCleanWS()
                }
                sh "git clone -b ${SCM_GIT_BRANCH} ${SCM_GIT_REPO_VAL} aqa-tests"
            }
        } else {
            def gitConfig = scm.getUserRemoteConfigs().get(0)

            // Adopt windows machines require env here https://github.com/adoptium/aqa-tests/issues/1803
            ref_cache = "${env.HOME}/openjdk_cache"

            retry(5) {
                if (retry_count > 0) {
                    sleep(sleep_time)
                }
                retry_count++
                timeout(time: 1, unit: 'HOURS') {
                    forceCleanWS()
                }
                checkout scm: [$class: 'GitSCM',
                    branches: [[name: "${scm.branches[0].name}"]],
                    extensions: [
                        [$class: 'CleanBeforeCheckout'],
                        [$class: 'CloneOption', reference: ref_cache],
                        [$class: 'RelativeTargetDirectory', relativeTargetDir: 'aqa-tests']],
                    userRemoteConfigs: [[url: "${gitConfig.getUrl()}"]]
                ]
            }
        }
        jenkinsfile = load "${WORKSPACE}/aqa-tests/buildenv/jenkins/JenkinsfileBase"
        if (LABEL.contains('ci.agent.dynamic') && CLOUD_PROVIDER.equals('azure')) {
            //Set dockerimage for azure agent. Fyre has stencil to setup the right environment
            docker.image('adoptopenjdk/centos7_build_image').pull()
            docker.image('adoptopenjdk/centos7_build_image').inside {
                jenkinsfile.testBuild()
            }
        } else if (dockerAgentLabel.equals('default') && LABEL.contains('&&sw.tool.docker') && SPEC.equals('linux_riscv64')) {
            // The requested image's platform is (linux/amd64)
            docker.image('multiarch/qemu-user-static').pull()
            // Automatically registers foreign file formats with the kernel using binfmt_static to simplify execution of multi-architecture binaries and Docker containers
            // In fact, this command only needs to be executed once
            sh "docker run --rm --privileged multiarch/qemu-user-static --reset -p yes"
            docker.image('alibabadragonwell/riscv-normal-qemu_6.0.0-rvv-1.0:latest').pull()
            // The ${HOME} directory must be mounted, otherwise xvfb will fail to startup
            docker.image('alibabadragonwell/riscv-normal-qemu_6.0.0-rvv-1.0:latest').inside("-v ${env.HOME}:${env.HOME}") {
                jenkinsfile.testBuild()
            }
        } else {
            jenkinsfile.testBuild()
        }
    } catch (Exception e) {
        println("Exception: " + e.toString())
        // build result may not be updated correctly at the moment (see https://issues.jenkins.io/browse/JENKINS-56402)
        // if there is an exception, set currentBuild.result to ABORTED/FAILURE
        if (e.toString().contains("FlowInterruptedException")) {
            currentBuild.result = 'ABORTED'
        } else {
            currentBuild.result = 'FAILURE'
        }

    } finally {
        if (params.SLACK_CHANNEL) {
            timeout(time: 5, unit: 'MINUTES') {
                if (currentBuild.result == 'FAILURE') {
                    println("${env.JOB_NAME} #${env.BUILD_NUMBER} result is FAILURE. Checking console log for specific errors...")
                    // create error list and regex can be used
                    List<String> errorList = new ArrayList<String>();
                    errorList.add(".*There is not enough space in the file system.*");
                    errorList.add(".*No space left on device.*");
                    errorList.add(".*java.io.IOException: Cannot run program \"nohup\".*");
                    errorList.add(".*unzip: command not found.*");
                    errorList.add(".*ant: command not found.*");
                    errorList.add(".*ERROR: Cannot delete workspace.*");

                    // nightly/weekly builds should not have git clone issue
                    if (!env.JOB_NAME.contains("Grinder") && !env.JOB_NAME.contains("_Personal")) {
                        errorList.add(".*ERROR: Error cloning remote repo.*");
                    }
                    checkErrors(errorList)
                }
            }
        }
    }
}

private String resolvePlatform(PLATFORM, JDK_VERSION) {
    println "RESOLVING PLATFORM=$PLATFORM JDK_VERSION=$JDK_VERSION"
    List<String> all_platforms = ["ppc64_aix", "ppc64le_linux", "s390x_linux", "x86-64_linux", "x86-64_mac", "x86-64_windows"]
    if (JDK_VERSION == "8") {
        all_platforms <<= "x86-32_windows"
    } else {
        all_platforms <<= "aarch64_linux"
    }

    if (PLATFORM == "all") {
        return all_platforms.join(",")
    }

    // if platform matches "all_{os}" pattern, return all platforms of that OS
    def match = PLATFORM =~ /(?i)all_(?<os>\w+)/
    if (match && match.groupCount() > 0) {
        String matched_os = match.group("os")
        println "matched_os=$matched_os"
        return all_platforms.findAll { it.contains(matched_os) }.join(",")
    }

    // otherwise, return the platform we received as input (no expansion needed)
    return PLATFORM
}

def areNodesWithLabelOnline(labelToCheck) {
            String[] onlineNodes = nodesByLabel(labelToCheck)
            if (onlineNodes.size() < 1) {
                return false
            }
            return true
        }


def checkErrors(errorList) {
    def foundError = ""
    for (String error : errorList) {
        if (manager.logContains(error)) {
            foundError = error;
            break;
        }
    }
    if (foundError) {
        def message = "${env.JOB_NAME} #${env.BUILD_NUMBER} failed on ${NODE_NAME} due to ${foundError}: ${env.BUILD_URL} \nSet ${NODE_NAME} temporarily offline! ${env.JENKINS_URL}computer/${NODE_NAME}"
        def currentNode = Jenkins.instance.getNode(NODE_NAME).toComputer()
        // set the current node temporarily offline
        if (currentNode) {
            println("${message}")
            currentNode.setTemporarilyOffline(true, new hudson.slaves.OfflineCause.UserCause(User.current(), "${message}"))
        }
        // if SLACK_CHANNEL is provided, send the message in slack
        if (params.SLACK_CHANNEL) {
            slackSend channel: SLACK_CHANNEL, message: "${message}"
        }
    }
}


def forceCleanWS() {
    try {
        cleanWs disableDeferredWipeout: true, deleteDirs: true
    } catch (Exception e) {
        echo 'Exception: ' + e.toString()
        //cleanWs has issue to delete workspace that contains non-ASCII filename in TKG output https://issues.jenkins.io/browse/JENKINS-33478
        //cannot delete workspace directly. Otherwise, Jenkins job will abort due to missing workspace
        sh "rm -rf ${env.WORKSPACE}/aqa-tests/TKG"
        // call cleanWs() again
        cleanWs disableDeferredWipeout: true, deleteDirs: true
    }
}

def changeParam(paramsList, newParams) {
    def childParams = []
    // loop through all the params and change the parameters if needed
    paramsList.each { param ->
        def value = param.value.toString()
        newParams.each { newParam ->

            if (param.key == newParam.key) {
                value = newParam.value.toString()
            }
        }
        if (value == "true" || value == "false") {
            childParams << booleanParam(name: param.key, value: value.toBoolean())
        } else {
            childParams << string(name: param.key, value: value)
        }
    }
    return childParams
}

def generateJobViaAutoGen(testJobName) {
    def JobHelper = library(identifier: 'openjdk-jenkins-helper@master').JobHelper
    if (params.GENERATE_JOBS || !JobHelper.jobIsRunnable(testJobName as String)) {
        def jobParams = []
        jobParams << string(name: 'TEST_JOB_NAME', value: testJobName)
        jobParams << string(name: 'ARCH_OS_LIST', value: params.PLATFORM)
        jobParams << booleanParam(name: 'LIGHT_WEIGHT_CHECKOUT', value: false)

        def jdk_impl = params.JDK_IMPL ?: ""
        def jdk_version = params.JDK_VERSION ?: ""
        if (testJobName.startsWith("Test_openjdk")) {
            def tokens = testJobName.split('_');
            def level = ""
            def group = ""
            if (tokens.size() > 3) {
                if (!jdk_version) {
                    jdk_version = tokens[1]
                }
                if (!jdk_impl) {
                    jdk_impl = tokens[2]
                }
                if (tokens[3].contains(".")) {
                    level = tokens[3].split("\\.")[0]
                    group = tokens[3].split("\\.")[1]
                    if (level && group) {
                        jobParams << string(name: 'LEVELS', value: level)
                        jobParams << string(name: 'GROUPS', value: group)
                    }
                }
            }
        }
        jobParams << string(name: 'JDK_IMPL', value: jdk_impl)
        jobParams << string(name: 'JDK_VERSIONS', value: jdk_version)

        build job: 'Test_Job_Auto_Gen', parameters: jobParams, propagate: true
    }
}