diff --git a/jenkins/opensearch/integ-test.jenkinsfile b/jenkins/opensearch/integ-test.jenkinsfile index f12b5a13cb..8d431f2480 100644 --- a/jenkins/opensearch/integ-test.jenkinsfile +++ b/jenkins/opensearch/integ-test.jenkinsfile @@ -1,8 +1,23 @@ -lib = library(identifier: 'jenkins@1.0.4', retriever: modernSCM([ +lib = library(identifier: 'jenkins@2.1.0', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', ])) +def docker_images = [ + "tar": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2", + "rpm": "opensearchstaging/ci-runner:ci-runner-rockylinux8-systemd-base-integtest-v1", +] + +def docker_args = [ + "tar": "-u 1000", + "rpm": "--entrypoint=/usr/sbin/init -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro" +] + +def agent_nodes = [ + "x64": "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host", + "arm64": "Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host", +] + pipeline { options { timeout(time: 3, unit: 'HOURS') @@ -10,61 +25,62 @@ pipeline { agent none environment { BUILD_MANIFEST = "build-manifest.yml" - DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch" + BUILD_JOB_NAME = "distribution-build-opensearch" ARTIFACT_BUCKET_NAME = credentials('jenkins-artifact-bucket-name') } parameters { string( - name: 'TEST_MANIFEST', - description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml.', + name: 'COMPONENT_NAME', + description: 'If this field contains one or more component names (e.g. index-management geospatial ...) separated by space, will test with "--component ...", else test everything in the TEST_MANIFEST..', trim: true ) string( - name: 'BUILD_MANIFEST_URL', - description: 'The build manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.2/98/linux/x64/tar/builds/opensearch/manifest.yml.', + name: 'TEST_MANIFEST', + description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml.', trim: true ) string( - name: 'AGENT_LABEL', - description: 'The agent label where the tests should be executed. For x64 use Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host,for arm64 use Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host', + name: 'BUILD_MANIFEST_URL', + description: 'The build manifest URL, e.g. "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.5.0/6976/linux/x64/tar/builds/opensearch/manifest.yml".', trim: true ) } stages { stage('verify-parameters') { - agent { label AGENT_LABEL } + agent { label agent_nodes["x64"]} steps { script { - currentBuild.description = BUILD_MANIFEST_URL - if (AGENT_LABEL == '') { - currentBuild.result = 'ABORTED' - error("Integration Tests failed to start. Missing parameter: AGENT_LABEL.") - } if (TEST_MANIFEST == '' || !fileExists("manifests/${TEST_MANIFEST}")) { currentBuild.result = 'ABORTED' error("Integration Tests failed to start. Test manifest was not provided or not found in manifests/${TEST_MANIFEST}.") } - /* - Rebuilding of this job will result in considering upstream build as self($JOB_NAME) See https://issues.jenkins.io/browse/JENKINS-61590 for bug - Either trigger from expected upstream job or run a new build - */ - env.BUILD_JOB_NAME = currentBuild.upstreamBuilds ? - currentBuild.upstreamBuilds[0].fullProjectName : - env.DEFAULT_BUILD_JOB_NAME - } - } - } - stage('detect docker image + args') { - agent { - docker { - label AGENT_LABEL - image 'alpine:3' - alwaysPull true + + if (BUILD_MANIFEST_URL == '') { + currentBuild.result = 'ABORTED' + error("Integration Tests failed to start. Build manifest url was not provided.") + } + + downloadBuildManifest( + url: BUILD_MANIFEST_URL, + path: BUILD_MANIFEST + ) + + def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: BUILD_MANIFEST)) + def componentList = COMPONENT_NAME ? COMPONENT_NAME.trim().split(" ") as List : buildManifestObj.getNames() + env.architecture = buildManifestObj.getArtifactArchitecture() + env.buildId = buildManifestObj.getArtifactBuildId() + env.distribution = buildManifestObj.getDistribution() + env.version = buildManifestObj.build.version + env.artifactPath = buildManifestObj.getArtifactRoot(BUILD_JOB_NAME, buildId) + env.AGENT_LABEL = agent_nodes["$architecture"] + + echo "Version: ${version}, Agent: ${AGENT_LABEL}, BuildId: ${buildId}, Distribution: ${distribution}, Components: ${componentList}" + currentBuild.description = "$version, $architecture, $buildId, $distribution, $componentList" } } - steps { - script { - DOCKER_AGENT = detectTestDockerAgent() + post { + always { + postCleanup() } } } @@ -80,55 +96,67 @@ pipeline { path: BUILD_MANIFEST ) - // Stash the current working directory files, aka opensearch-build repo - // Unstash later in each triggered stage to run integTest - stash includes: '**', name: 'opensearch-build-repo' - def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: BUILD_MANIFEST)) - def componentList = buildManifestObj.getNames() - String distribution = buildManifestObj.getDistribution() - String buildId = buildManifestObj.getArtifactBuildId() - String artifactPath = buildManifestObj.getArtifactRoot(BUILD_JOB_NAME, buildId) + def componentDefaultList = buildManifestObj.getNames() + def componentList = COMPONENT_NAME ? COMPONENT_NAME.trim().split(" ") as List : componentDefaultList + String switch_user_non_root = (distribution.equals('rpm') || distribution.equals('deb')) ? 'true' : 'false' + echo "switch_user_non_root: ${switch_user_non_root}" echo "componentList: ${componentList}" + for (component_check in componentList) { + if (! componentDefaultList.contains(component_check)) { + error("${component_check} is not in build manifest: ${componentDefaultList}, exit 1") + } + } + + echo "Downloading from S3: ${artifactPath}" + downloadFromS3( + assumedRoleName: 'opensearch-bundle', + roleAccountNumberCred: 'jenkins-aws-account-public', + downloadPath: "${artifactPath}/", + bucketName: "${ARTIFACT_BUCKET_NAME}", + localPath: "${WORKSPACE}/artifacts", + force: true + ) + sh("mv -v $WORKSPACE/artifacts/${artifactPath} $WORKSPACE") + + // Stash the current working directory files, aka opensearch-build repo + // Unstash later in each triggered stage to run integTest + stash includes: "**", name: "opensearch-build-repo-$BUILD_NUMBER" + componentTests = [:] for (component in componentList) { // Must use local variable due to groovy for loop and closure scope // Or the stage will be fixed to the last item in return when new stages are triggered here // https://web.archive.org/web/20181121065904/http://blog.freeside.co/2013/03/29/groovy-gotcha-for-loops-and-closure-scope/ - def local_component = component - def wait_seconds = componentList.indexOf(local_component) * 20 + def local_component = component.trim() + def local_component_index = componentList.indexOf(local_component) + def wait_seconds = local_component_index * 10 echo "Add Component: ${local_component}" componentTests["Run Integtest ${local_component}"] = { // Using scripted pipelines to trigger dynamic parallel stages timeout(time: 2, unit: 'HOURS') { node(AGENT_LABEL) { - docker.image(DOCKER_AGENT.image).inside(DOCKER_AGENT.args) { + docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) { try { stage("Run Integtest ${local_component}") { echo "Component Name: ${local_component}" - unstash 'opensearch-build-repo' // Jenkins tend to not clean up workspace at times even though ws clean is called // Due to docker is mounting the agent directory so it can communicated with the agent // This sometimes causes the workspace to retain last run test-results and ends with build failures // https://github.com/opensearch-project/opensearch-build/blob/6ed1ce3c583233eae4fe1027969d778cfc7660f7/src/test_workflow/test_recorder/test_recorder.py#L99 - sh("rm -rf test-results ${WORKSPACE}/${distribution} && echo sleep ${wait_seconds} seconds && sleep ${wait_seconds}") - echo "Downloading from S3: ${artifactPath}" - downloadFromS3( - destPath: "$WORKSPACE/artifacts", - bucket: "${ARTIFACT_BUCKET_NAME}", - path: "${artifactPath}/", - force: true - ) - sh("mv -v $WORKSPACE/artifacts/${artifactPath} $WORKSPACE") + sh("echo ${local_component} with index ${local_component_index} will sleep ${wait_seconds} seconds to reduce load && sleep ${wait_seconds}") + unstash "opensearch-build-repo-$BUILD_NUMBER" + sh("rm -rf test-results") runIntegTestScript( - jobName: BUILD_JOB_NAME, + jobName: "$BUILD_JOB_NAME", componentName: "${local_component}", - buildManifest: BUILD_MANIFEST, + buildManifest: "$BUILD_MANIFEST", testManifest: "manifests/${TEST_MANIFEST}", - localPath: "${WORKSPACE}/${distribution}" + localPath: "${WORKSPACE}/${distribution}", + switchUserNonRoot: "${switch_user_non_root}" ) } } catch (e) { diff --git a/tests/jenkins/TestOpenSearchIntegTest.groovy b/tests/jenkins/TestOpenSearchIntegTest.groovy index 3192fde80f..4c61460510 100644 --- a/tests/jenkins/TestOpenSearchIntegTest.groovy +++ b/tests/jenkins/TestOpenSearchIntegTest.groovy @@ -19,7 +19,7 @@ class TestOpenSearchIntegTest extends BuildPipelineTest { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('1.0.4') + .defaultVersion('2.1.0') .allowOverride(true) .implicit(true) .targetPath('vars') diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt index b2f5b7fb57..c732ab4248 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt @@ -1,6 +1,6 @@ integ-test.run() integ-test.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - integ-test.library({identifier=jenkins@1.0.4, retriever=null}) + integ-test.library({identifier=jenkins@2.1.0, retriever=null}) integ-test.pipeline(groovy.lang.Closure) integ-test.credentials(jenkins-artifact-bucket-name) integ-test.timeout({time=3, unit=HOURS}) @@ -9,62 +9,80 @@ integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) integ-test.script(groovy.lang.Closure) integ-test.fileExists(manifests/tests/jenkins/data/opensearch-1.3.0-test.yml) - integ-test.stage(detect docker image + args, groovy.lang.Closure) - integ-test.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:alpine:3, reuseNode:false, stages:[:]]]) - integ-test.script(groovy.lang.Closure) - integ-test.detectTestDockerAgent() - detectTestDockerAgent.legacySCM(groovy.lang.Closure) - detectTestDockerAgent.library({identifier=jenkins@1.0.4, retriever=null}) - detectTestDockerAgent.readYaml({file=manifests/tests/jenkins/data/opensearch-1.3.0-test.yml}) - TestManifest.asBoolean() - detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) + integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-1.3.0-build.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@2.1.0, retriever=null}) + downloadBuildManifest.sh(curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz --output tests/jenkins/data/opensearch-1.3.0-build.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml}) + BuildManifest.asBoolean() + integ-test.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml}) + BuildManifest.asBoolean() + BuildManifest.getNames() + BuildManifest.getArtifactArchitecture() + BuildManifest.getArtifactBuildId() + BuildManifest.getDistribution() + BuildManifest.getArtifactRoot(distribution-build-opensearch, 717) + integ-test.echo(Version: 1.3.0, Agent: Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, BuildId: 717, Distribution: tar, Components: [OpenSearch]) + integ-test.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) integ-test.stage(integ-test, groovy.lang.Closure) integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) integ-test.script(groovy.lang.Closure) integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-1.3.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@1.0.4, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@2.1.0, retriever=null}) downloadBuildManifest.sh(curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz --output tests/jenkins/data/opensearch-1.3.0-build.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml}) BuildManifest.asBoolean() - integ-test.stash({includes=**, name=opensearch-build-repo}) integ-test.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml}) BuildManifest.asBoolean() BuildManifest.getNames() - BuildManifest.getDistribution() - BuildManifest.getArtifactBuildId() - BuildManifest.getArtifactRoot(dummy_job, 717) + integ-test.echo(switch_user_non_root: false) integ-test.echo(componentList: [OpenSearch]) + integ-test.echo(Downloading from S3: distribution-build-opensearch/1.3.0/717/linux/x64/tar) + integ-test.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=distribution-build-opensearch/1.3.0/717/linux/x64/tar/, bucketName=job-s3-bucket-name, localPath=/tmp/workspace/artifacts, force=true}) + downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_NUMBER}) + downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure) + downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure) + downloadFromS3.s3Download({file=/tmp/workspace/artifacts, bucket=job-s3-bucket-name, path=distribution-build-opensearch/1.3.0/717/linux/x64/tar/, force=true}) + integ-test.sh(mv -v /tmp/workspace/artifacts/distribution-build-opensearch/1.3.0/717/linux/x64/tar /tmp/workspace) + integ-test.stash({includes=**, name=opensearch-build-repo-717}) integ-test.echo(Add Component: OpenSearch) integ-test.parallel({Run Integtest OpenSearch=groovy.lang.Closure}) integ-test.timeout({time=2, unit=HOURS}, groovy.lang.Closure) integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) integ-test.echo(Component Name: OpenSearch) - integ-test.unstash(opensearch-build-repo) - integ-test.sh(rm -rf test-results /tmp/workspace/tar && echo sleep 0 seconds && sleep 0) - integ-test.echo(Downloading from S3: dummy_job/1.3.0/717/linux/x64/tar) - integ-test.downloadFromS3({destPath=/tmp/workspace/artifacts, bucket=job-s3-bucket-name, path=dummy_job/1.3.0/717/linux/x64/tar/, force=true}) - downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) - downloadFromS3.withCredentials([AWS_ACCOUNT_PUBLIC], groovy.lang.Closure) - downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - downloadFromS3.s3Download({file=/tmp/workspace/artifacts, bucket=job-s3-bucket-name, path=dummy_job/1.3.0/717/linux/x64/tar/, force=true}) - integ-test.sh(mv -v /tmp/workspace/artifacts/dummy_job/1.3.0/717/linux/x64/tar /tmp/workspace) - integ-test.runIntegTestScript({jobName=dummy_job, componentName=OpenSearch, buildManifest=tests/jenkins/data/opensearch-1.3.0-build.yml, testManifest=manifests/tests/jenkins/data/opensearch-1.3.0-test.yml, localPath=/tmp/workspace/tar}) + integ-test.sh(echo OpenSearch with index 0 will sleep 0 seconds to reduce load && sleep 0) + integ-test.unstash(opensearch-build-repo-717) + integ-test.sh(rm -rf test-results) + integ-test.runIntegTestScript({jobName=distribution-build-opensearch, componentName=OpenSearch, buildManifest=tests/jenkins/data/opensearch-1.3.0-build.yml, testManifest=manifests/tests/jenkins/data/opensearch-1.3.0-test.yml, localPath=/tmp/workspace/tar, switchUserNonRoot=false}) runIntegTestScript.legacySCM(groovy.lang.Closure) - runIntegTestScript.library({identifier=jenkins@1.0.4, retriever=null}) + runIntegTestScript.library({identifier=jenkins@2.1.0, retriever=null}) runIntegTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml}) BuildManifest.asBoolean() + BuildManifest.getDistribution() + runIntegTestScript.echo(Start integTest for distribution type: tar) + runIntegTestScript.detectTestDockerAgent() + detectTestDockerAgent.legacySCM(groovy.lang.Closure) + detectTestDockerAgent.library({identifier=jenkins@2.1.0, retriever=null}) + detectTestDockerAgent.readYaml({file=manifests/tests/jenkins/data/opensearch-1.3.0-test.yml}) + TestManifest.asBoolean() + detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) + detectTestDockerAgent.echo(Using java version openjdk-17) + runIntegTestScript.echo(Possible Java Home: env JAVA_HOME=/opt/java/openjdk-17) runIntegTestScript.echo(Build Id: 717) - BuildManifest.getArtifactRootUrl(dummy_job, 717) - runIntegTestScript.echo(Artifact root URL: https://ci.opensearch.org/ci/dbc/dummy_job/1.3.0/717/linux/x64/tar) + BuildManifest.getArtifactRootUrl(distribution-build-opensearch, 717) + runIntegTestScript.echo(Artifact root URL: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/tar) runIntegTestScript.echo(User provides localPath, use local artifacts: /tmp/workspace/tar) runIntegTestScript.echo(Paths: opensearch=/tmp/workspace/tar) runIntegTestScript.echo(Component: OpenSearch) - runIntegTestScript.sh(./test.sh integ-test manifests/tests/jenkins/data/opensearch-1.3.0-test.yml --component OpenSearch --test-run-id 717 --paths opensearch=/tmp/workspace/tar) + runIntegTestScript.echo(Switch User to Non-Root (uid=1000): false) + runIntegTestScript.echo(Run command: env JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-1.3.0-test.yml --component OpenSearch --test-run-id 717 --paths opensearch=/tmp/workspace/tar ) + runIntegTestScript.sh( env JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-1.3.0-test.yml --component OpenSearch --test-run-id 717 --paths opensearch=/tmp/workspace/tar ) integ-test.echo(Completed running integtest for component OpenSearch) integ-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) - uploadTestResults.library({identifier=jenkins@1.0.4, retriever=null}) + uploadTestResults.library({identifier=jenkins@2.1.0, retriever=null}) uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml}) BuildManifest.asBoolean() uploadTestResults.echo(Build Id: 717)