diff --git a/src/test/groovy/ApmBasePipelineTest.groovy b/src/test/groovy/ApmBasePipelineTest.groovy index feb3430f2..6f1788c9e 100644 --- a/src/test/groovy/ApmBasePipelineTest.groovy +++ b/src/test/groovy/ApmBasePipelineTest.groovy @@ -63,12 +63,13 @@ class ApmBasePipelineTest extends BasePipelineTest { helper.registerAllowedMethod('failFast', [Boolean.class], null) helper.registerAllowedMethod('issueCommentTrigger', [String.class], null) helper.registerAllowedMethod('label', [String.class], null) - helper.registerAllowedMethod('options', [Closure.class], null) + helper.registerAllowedMethod('options', [Closure.class], { body -> body() }) helper.registerAllowedMethod('pipeline', [Closure.class], null) helper.registerAllowedMethod('post', [Closure.class], null) helper.registerAllowedMethod('quietPeriod', [Integer.class], null) helper.registerAllowedMethod('rateLimitBuilds', [Map.class], null) helper.registerAllowedMethod('script', [Closure.class], { body -> body() }) + helper.registerAllowedMethod('skipDefaultCheckout', [], null) helper.registerAllowedMethod('stage', [Closure.class], null) helper.registerAllowedMethod('stage', [String.class, Closure.class], { stageName, body -> def stageResult @@ -79,6 +80,21 @@ class ApmBasePipelineTest extends BasePipelineTest { } throw new RuntimeException("Stage \"${stageName}\" skipped due to when conditional") }) + helper.registerAllowedMethod('tag', [String.class], { tagName -> + // Default comparator = EQUALS in this particular implementation + if(tagName == env.BRANCH_NAME) { + return true + } + throw new RuntimeException("Stage \"${stageName}\" skipped due to when conditional") + }) + helper.registerAllowedMethod('tag', [Map.class], { m -> + if (m.comparator.equals('REGEXP')) { + if (env.BRANCH_NAME ==~ m.pattern) { + return true + } + } + throw new RuntimeException("Stage \"${stageName}\" skipped due to when conditional") + }) helper.registerAllowedMethod('allOf', [Closure.class], { Closure cAllOf -> helper.registerAllowedMethod('branch', [String.class], { branchName -> if(branchName == env.BRANCH_NAME) { @@ -94,6 +110,27 @@ class ApmBasePipelineTest extends BasePipelineTest { }) return cAllOf() }) + helper.registerAllowedMethod('anyOf', [Closure.class], { Closure cAnyOf -> + def result = false + helper.registerAllowedMethod('branch', [String.class], { branchName -> + if(branchName == env.BRANCH_NAME) { + result = true + return result + } + }) + helper.registerAllowedMethod('tag', [Map.class], { m -> + if (m.comparator.equals('REGEXP')) { + if (env.BRANCH_NAME ==~ m.pattern) { + result = true + return result + } + } + if (!result) { + throw new RuntimeException("Stage \"${stageName}\" skipped due to when conditional (branch)") + } + }) + return cAnyOf() + }) return bodyWhen() }) @@ -128,7 +165,7 @@ class ApmBasePipelineTest extends BasePipelineTest { helper.registerAllowedMethod('bat', [String.class], null) helper.registerAllowedMethod('brokenTestsSuspects', { "OK" }) helper.registerAllowedMethod('brokenBuildSuspects', { "OK" }) - helper.registerAllowedMethod('upstreamDevelopers', { "OK" }) + helper.registerAllowedMethod('build', [Map.class], null) helper.registerAllowedMethod('catchError', [Closure.class], { c -> try{ c() @@ -193,12 +230,13 @@ class ApmBasePipelineTest extends BasePipelineTest { c.call() }) helper.registerAllowedMethod('sleep', [Integer.class], null) - helper.registerAllowedMethod("sh", [Map.class], { 'OK' }) + helper.registerAllowedMethod('sh', [Map.class], { 'OK' }) helper.registerAllowedMethod('sh', [String.class], { 'OK' }) helper.registerAllowedMethod('sshagent', [List.class, Closure.class], { m, body -> body() }) helper.registerAllowedMethod('string', [Map.class], { m -> return m }) helper.registerAllowedMethod('timeout', [Integer.class, Closure.class], null) helper.registerAllowedMethod('unstash', [String.class], null) + helper.registerAllowedMethod('upstreamDevelopers', { "OK" }) helper.registerAllowedMethod('usernamePassword', [Map.class], { m -> m.each{ k, v -> binding.setVariable("${v}", 'defined') diff --git a/src/test/groovy/OpbeansPipelineStepTests.groovy b/src/test/groovy/OpbeansPipelineStepTests.groovy index d2c2cb8ec..65d5fdf8f 100644 --- a/src/test/groovy/OpbeansPipelineStepTests.groovy +++ b/src/test/groovy/OpbeansPipelineStepTests.groovy @@ -18,6 +18,7 @@ import org.junit.Before import org.junit.Test import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.junit.Assert.assertEquals import static org.junit.Assert.assertFalse import static org.junit.Assert.assertNull import static org.junit.Assert.assertTrue @@ -30,6 +31,8 @@ class OpbeansPipelineStepTests extends ApmBasePipelineTest { void setUp() throws Exception { binding.setProperty('BASE_DIR', '/') binding.setProperty('DOCKERHUB_SECRET', 'secret') + env.GIT_BASE_COMMIT = '1' + env.REPO_NAME = 'opbeans-xyz' super.setUp() } @@ -48,7 +51,9 @@ class OpbeansPipelineStepTests extends ApmBasePipelineTest { assertTrue(helper.callStack.findAll { call -> call.methodName == 'stage' }.any { call -> callArgsToString(call).contains('Release') }) - assertNull(helper.callStack.find { call -> call.methodName == 'build' }) + assertTrue((helper.callStack.findAll { call -> call.methodName == 'build' } - + helper.callStack.findAll { call -> call.methodName == 'build' }.findAll { call -> + callArgsToString(call).contains('job=apm-integration-tests-selector-mbp/master')}).isEmpty()) assertJobStatusSuccess() } @@ -58,8 +63,9 @@ class OpbeansPipelineStepTests extends ApmBasePipelineTest { env.BRANCH_NAME = 'master' script.call(downstreamJobs: []) printCallStack() - assertNull(helper.callStack.find { call -> call.methodName == 'build' }) - assertJobStatusSuccess() + assertTrue((helper.callStack.findAll { call -> call.methodName == 'build' } - + helper.callStack.findAll { call -> call.methodName == 'build' }.findAll { call -> + callArgsToString(call).contains('job=apm-integration-tests-selector-mbp/master')}).isEmpty()) } @Test @@ -74,6 +80,9 @@ class OpbeansPipelineStepTests extends ApmBasePipelineTest { assertTrue(helper.callStack.findAll { call -> call.methodName == 'build' }.any { call -> callArgsToString(call).contains('folder/foo') }) + assertTrue(helper.callStack.findAll { call -> call.methodName == 'sh' }.any { call -> + callArgsToString(call).contains('VERSION=latest make publish') + }) assertJobStatusSuccess() } @@ -91,4 +100,85 @@ class OpbeansPipelineStepTests extends ApmBasePipelineTest { assertJobStatusSuccess() } + @Test + void test_when_tag_release() throws Exception { + def script = loadScript(scriptName) + // When the tag release does match + env.BRANCH_NAME = 'v1.0' + script.call() + printCallStack() + // Then publish shell step + assertTrue(helper.callStack.findAll { call -> call.methodName == 'sh' }.any { call -> + callArgsToString(call).contains('VERSION=agent-v1.0 make publish') + }) + assertJobStatusSuccess() + } + + @Test + void test_getForkedRepoOrElasticRepo() throws Exception { + def script = loadScript(scriptName) + env.CHANGE_FORK = 'user/forked_repo' + def result = script.getForkedRepoOrElasticRepo('foo') + assertEquals(result, 'user/forked_repo') + assertJobStatusSuccess() + } + + @Test + void test_getForkedRepoOrElasticRepo_without_change_fork() throws Exception { + def script = loadScript(scriptName) + def result = script.getForkedRepoOrElasticRepo('repo') + assertEquals(result, 'elastic/repo') + assertJobStatusSuccess() + } + + @Test + void test_getForkedRepoOrElasticRepo_with_change_fork() throws Exception { + def script = loadScript(scriptName) + env.CHANGE_FORK = 'user' + def result = script.getForkedRepoOrElasticRepo('repo') + assertEquals(result, 'user/repo') + assertJobStatusSuccess() + } + + @Test + void test_generateBuildOpts_without_known_repo() throws Exception { + def script = loadScript(scriptName) + def result = script.generateBuildOpts('unknown', '') + assertEquals(result, '') + assertJobStatusSuccess() + } + + @Test + void test_generateBuildOpts_with_go() throws Exception { + def script = loadScript(scriptName) + def result = script.generateBuildOpts('opbeans-go', '') + assertEquals(result, '--with-opbeans-go --opbeans-go-branch 1 --opbeans-go-repo elastic/opbeans-go') + assertJobStatusSuccess() + } + + @Test + void test_generateBuildOpts_with_go_and_forked_repo() throws Exception { + def script = loadScript(scriptName) + env.CHANGE_FORK = 'user' + def result = script.generateBuildOpts('opbeans-go', '') + assertEquals(result, '--with-opbeans-go --opbeans-go-branch 1 --opbeans-go-repo user/opbeans-go') + assertJobStatusSuccess() + } + + @Test + void test_generateBuildOpts_with_java() throws Exception { + def script = loadScript(scriptName) + def result = script.generateBuildOpts('opbeans-java', 'foo') + assertEquals(result, '--with-opbeans-java --opbeans-java-image foo --opbeans-java-version 1') + assertJobStatusSuccess() + } + + @Test + void test_waitIfNotPR() throws Exception { + def script = loadScript(scriptName) + assertTrue(script.waitIfNotPR()) + env.CHANGE_ID = 'PR-1' + assertFalse(script.waitIfNotPR()) + assertJobStatusSuccess() + } } diff --git a/vars/agentMapping.groovy b/vars/agentMapping.groovy index 4f919702c..911a1331d 100644 --- a/vars/agentMapping.groovy +++ b/vars/agentMapping.groovy @@ -82,6 +82,7 @@ import groovy.transform.Field 'Ruby': 'ruby', 'RUM': 'rum', 'All': 'all', + 'Opbeans': '', // This is required for getting the docker logs 'UI': 'ui' ] diff --git a/vars/opbeansPipeline.groovy b/vars/opbeansPipeline.groovy index 934f824f8..c27ec7007 100644 --- a/vars/opbeansPipeline.groovy +++ b/vars/opbeansPipeline.groovy @@ -35,6 +35,11 @@ def call(Map pipelineParams) { PIPELINE_LOG_LEVEL = 'INFO' PATH = "${env.PATH}:${env.WORKSPACE}/bin" HOME = "${env.WORKSPACE}" + DOCKER_REGISTRY_SECRET = 'secret/apm-team/ci/docker-registry/prod' + REGISTRY = 'docker.elastic.co' + STAGING_IMAGE = "${env.REGISTRY}/observability-ci" + GITHUB_CHECK_ITS_NAME = 'Integration Tests' + ITS_PIPELINE = 'apm-integration-tests-selector-mbp/master' } options { timeout(time: 1, unit: 'HOURS') @@ -47,7 +52,7 @@ def call(Map pipelineParams) { quietPeriod(10) } triggers { - issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*') + issueCommentTrigger('(?i).*jenkins\\W+run\\W+(?:the\\W+)?tests(?:\\W+please)?.*') } stages { /** @@ -83,7 +88,7 @@ def call(Map pipelineParams) { deleteDir() unstash 'source' dir(BASE_DIR){ - sh "make test" + sh 'make test' } } } @@ -95,23 +100,24 @@ def call(Map pipelineParams) { } } } - stage('Release') { - agent { label 'linux && immutable' } - when { - branch 'master' - beforeAgent true - } + stage('Staging') { steps { - withGithubNotify(context: 'Release') { + withGithubNotify(context: 'Staging') { deleteDir() unstash 'source' dir(BASE_DIR){ - dockerLogin(secret: "${DOCKERHUB_SECRET}", registry: 'docker.io') - sh "VERSION=latest make publish" + dockerLogin(secret: "${DOCKER_REGISTRY_SECRET}", registry: "${REGISTRY}") + sh label: "push docker image to ${env.STAGING_IMAGE}/${env.REPO_NAME}", + script: "VERSION=${env.GIT_BASE_COMMIT} IMAGE=${env.STAGING_IMAGE}/${env.REPO_NAME} make publish" } } } } + stage('Integration Tests') { + steps { + runBuildITs("${env.REPO_NAME}", "${env.STAGING_IMAGE}/${env.REPO_NAME}") + } + } stage('Downstream') { when { allOf { @@ -128,6 +134,39 @@ def call(Map pipelineParams) { } } } + stage('Release') { + when { + anyOf { + branch 'master' + tag pattern: 'v\\d+\\.\\d+.*', comparator: 'REGEXP' + } + } + environment { + VERSION = "${env.BRANCH_NAME.equals('master') ? 'latest' : 'agent-' + env.BRANCH_NAME}" + } + stages { + stage('Publish') { + steps { + withGithubNotify(context: 'Publish') { + deleteDir() + unstash 'source' + dir(BASE_DIR){ + dockerLogin(secret: "${DOCKERHUB_SECRET}", registry: 'docker.io') + sh "VERSION=${env.VERSION} make publish" + } + } + } + } + stage('Release Notes') { + when { + expression { return false } + } + steps { + echo 'TBD' + } + } + } + } } post { always { @@ -136,3 +175,42 @@ def call(Map pipelineParams) { } } } + +def runBuildITs(String repo, String stagingDockerImage) { + build(job: env.ITS_PIPELINE, propagate: waitIfNotPR(), + wait: env.CHANGE_ID?.trim() ? false : true, + parameters: [string(name: 'AGENT_INTEGRATION_TEST', value: 'Opbeans'), + string(name: 'BUILD_OPTS', value: "${generateBuildOpts(repo, stagingDockerImage)}"), + string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_ITS_NAME), + string(name: 'GITHUB_CHECK_REPO', value: repo), + string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT)]) + githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}") +} + +def generateBuildOpts(String repo, String stagingDockerImage) { + switch(repo) { + case 'opbeans-go': + opts = "--with-opbeans-go --opbeans-go-branch ${env.GIT_BASE_COMMIT} --opbeans-go-repo ${getForkedRepoOrElasticRepo(repo)}" + break; + case 'opbeans-java': + opts = "--with-opbeans-java --opbeans-java-image ${stagingDockerImage} --opbeans-java-version ${env.GIT_BASE_COMMIT}" + break; + default: + opts = '' + break; + } + return opts.toString() +} + +def waitIfNotPR() { + return env.CHANGE_ID?.trim() ? false : true +} + +def getForkedRepoOrElasticRepo(String repo) { + // See https://issues.jenkins-ci.org/browse/JENKINS-58450 + if (env.CHANGE_FORK?.contains('/')) { + return env.CHANGE_FORK + } else { + return "${env.CHANGE_FORK?.trim() ?: 'elastic' }/${repo}".toString() + } +}