From a7c1a9e938862d8e1460ee8fb7c2c76e17920e21 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Tue, 7 Feb 2017 14:05:42 -0800 Subject: [PATCH 01/10] Adds Jenkinsfile and update release-binary --- .bazelrc.jenkins | 3 +++ Jenkinsfile | 54 +++++++++++++++++++++++++++++++++++++++++++ script/release-binary | 15 +++++++----- 3 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 .bazelrc.jenkins create mode 100644 Jenkinsfile diff --git a/.bazelrc.jenkins b/.bazelrc.jenkins new file mode 100644 index 00000000000..c6b26c3e785 --- /dev/null +++ b/.bazelrc.jenkins @@ -0,0 +1,3 @@ +# This is so we understand failures better +build --verbose_failures + diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000000..811029b579b --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,54 @@ +#!groovy + +@Library('testutils') + +import org.istio.testutils.Utilities +import org.istio.testutils.GitUtilities +import org.istio.testutils.Bazel + +// Utilities shared amongst modules +def gitUtils = new GitUtilities() +def utils = new Utilities() +def bazel = new Bazel() + +node { + gitUtils.initialize() + bazel.setVars() +} + +mainFlow(utils) { + if (utils.runStage('PRESUBMIT')) { + def success = true + utils.updatePullRequest('run') + try { + presubmit(gitUtils, bazel) + } catch (Exception e) { + success = false + throw e + } finally { + utils.updatePullRequest('verify', success) + } + } + if (utils.runStage('RELEASE')) { + defaultNode(gitUtils) { + sh 'script/release-binary' + } + } +} + +def presubmit(gitUtils, bazel) { + goBuildNode(gitUtils, 'istio.io/manager') { + bazel.updateBazelRc() + stage('Code Check') { + sh('bin/check.sh') + } + stage('Bazel Build') { + sh('touch platform/kube/config') + bazel.fetch('-k //...') + bazel.build('//...') + } + stage('Bazel Tests') { + bazel.test('//...') + } + } +} diff --git a/script/release-binary b/script/release-binary index af90b1aa0da..bf83a7d3115 100755 --- a/script/release-binary +++ b/script/release-binary @@ -24,16 +24,19 @@ set -ex BUCKET_NAME="istio-build/proxy" # The proxy binary name. -BINARY_FORMAT='envoy-alpha-%H.tar.gz' -BINARY_NAME="$(git show -q HEAD --pretty=format:"${BINARY_FORMAT}")" +SHA="$(git rev-parse --verify HEAD)" +BINARY_NAME="envoy-alpha-${SHA}.tar.gz" +SHA256_NAME="envoy-alpha-${SHA}.sha256" # Build the binary bazel build --config=release //src/envoy/mixer:envoy_tar -SRC="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" -DST="gs://${BUCKET_NAME}/${BINARY_NAME}" +BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" +ln "${BAZEL_TARGET}" "${BINARY_NAME}" +sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" +DST="gs://${BUCKET_NAME}/" # Copy it to the bucket. -echo "Copying ${SRC} to ${DST}" -gsutil cp ${SRC} ${DST} +echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}" +gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}" From 3457136d360dac6bc9e7d8512db91a504d5172b3 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Tue, 7 Feb 2017 14:58:41 -0800 Subject: [PATCH 02/10] Update Jenkinsfile and gitignore --- .gitignore | 2 ++ Jenkinsfile | 12 ++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index a6ef824c1f8..469486c8f36 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /bazel-* +.idea/* +*.iml diff --git a/Jenkinsfile b/Jenkinsfile index 811029b579b..604dddb0590 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ mainFlow(utils) { utils.updatePullRequest('verify', success) } } - if (utils.runStage('RELEASE')) { + if (utils.runStage('POSTSUBMIT')) { defaultNode(gitUtils) { sh 'script/release-binary' } @@ -38,17 +38,21 @@ mainFlow(utils) { def presubmit(gitUtils, bazel) { goBuildNode(gitUtils, 'istio.io/manager') { - bazel.updateBazelRc() stage('Code Check') { sh('bin/check.sh') } - stage('Bazel Build') { - sh('touch platform/kube/config') + bazel.updateBazelRc() + stage('Bazel Fetch') { bazel.fetch('-k //...') + } + stage('Bazel Build') { bazel.build('//...') } stage('Bazel Tests') { bazel.test('//...') } + stage('Push Test Binary') { + sh 'script/release-binary' + } } } From 2f3b8afc6ecba8e5bb01d4905c3923878e31ff78 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Tue, 7 Feb 2017 15:44:16 -0800 Subject: [PATCH 03/10] Fixes typo and use normal build Node --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 604dddb0590..fb0b6aac1ec 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,9 +37,9 @@ mainFlow(utils) { } def presubmit(gitUtils, bazel) { - goBuildNode(gitUtils, 'istio.io/manager') { + buildNode(gitUtils) { stage('Code Check') { - sh('bin/check.sh') + sh('script/check-style') } bazel.updateBazelRc() stage('Bazel Fetch') { From e3a31a9aeec2a215252ffdff6e2a0597d73af3ef Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Tue, 7 Feb 2017 16:49:38 -0800 Subject: [PATCH 04/10] Uses default bazel config --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fb0b6aac1ec..c72e07d5c0d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,7 @@ def bazel = new Bazel() node { gitUtils.initialize() - bazel.setVars() + //bazel.setVars() } mainFlow(utils) { @@ -30,7 +30,7 @@ mainFlow(utils) { } } if (utils.runStage('POSTSUBMIT')) { - defaultNode(gitUtils) { + buildNode(gitUtils) { sh 'script/release-binary' } } From 1252d787d78c74e4c3ac1057131292e44c6d91cc Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Wed, 8 Feb 2017 14:06:11 -0800 Subject: [PATCH 05/10] Disabling bazel sandbox --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c72e07d5c0d..df989b6fcf7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,7 +46,7 @@ def presubmit(gitUtils, bazel) { bazel.fetch('-k //...') } stage('Bazel Build') { - bazel.build('//...') + bazel.build('--strategy=CppCompile=standalone //...') } stage('Bazel Tests') { bazel.test('//...') From 0d1ffd50f01b97128e481d19f3f66523451f4cda Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Wed, 8 Feb 2017 14:30:08 -0800 Subject: [PATCH 06/10] Using batch mode --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index df989b6fcf7..a2ed9245fe3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,7 +46,8 @@ def presubmit(gitUtils, bazel) { bazel.fetch('-k //...') } stage('Bazel Build') { - bazel.build('--strategy=CppCompile=standalone //...') + sh("bazel --batch build //...") + // bazel.build('--strategy=CppCompile=standalone //...') } stage('Bazel Tests') { bazel.test('//...') From 9a0bb6f6c9a95e756d06cda3631e7839b0094e78 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Wed, 8 Feb 2017 14:49:16 -0800 Subject: [PATCH 07/10] debugginb sandbox --- Jenkinsfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a2ed9245fe3..38259767541 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,15 +38,15 @@ mainFlow(utils) { def presubmit(gitUtils, bazel) { buildNode(gitUtils) { - stage('Code Check') { - sh('script/check-style') - } - bazel.updateBazelRc() - stage('Bazel Fetch') { - bazel.fetch('-k //...') - } + // stage('Code Check') { + // sh('script/check-style') + // } + // bazel.updateBazelRc() + // stage('Bazel Fetch') { + // bazel.fetch('-k //...') + // } stage('Bazel Build') { - sh("bazel --batch build //...") + sh("bazel --batch build --sandbox_debug //...") // bazel.build('--strategy=CppCompile=standalone //...') } stage('Bazel Tests') { From 0705c718d7db4c07ef525669ff1a900ac9e358b2 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Wed, 8 Feb 2017 15:22:41 -0800 Subject: [PATCH 08/10] Update bazel memory settings --- .bazelrc.jenkins | 4 ++++ Jenkinsfile | 19 +++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.bazelrc.jenkins b/.bazelrc.jenkins index c6b26c3e785..eda2c0fd2ee 100644 --- a/.bazelrc.jenkins +++ b/.bazelrc.jenkins @@ -1,3 +1,7 @@ +# This is from Bazel's former travis setup, to avoid blowing up the RAM usage. +startup --host_jvm_args=-Xmx8192m +startup --host_jvm_args=-Xms8192m + # This is so we understand failures better build --verbose_failures diff --git a/Jenkinsfile b/Jenkinsfile index 38259767541..904399033eb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,7 @@ def bazel = new Bazel() node { gitUtils.initialize() - //bazel.setVars() + bazel.setVars() } mainFlow(utils) { @@ -38,16 +38,15 @@ mainFlow(utils) { def presubmit(gitUtils, bazel) { buildNode(gitUtils) { - // stage('Code Check') { - // sh('script/check-style') - // } - // bazel.updateBazelRc() - // stage('Bazel Fetch') { - // bazel.fetch('-k //...') - // } + stage('Code Check') { + sh('script/check-style') + } + bazel.updateBazelRc() + stage('Bazel Fetch') { + bazel.fetch('-k //...') + } stage('Bazel Build') { - sh("bazel --batch build --sandbox_debug //...") - // bazel.build('--strategy=CppCompile=standalone //...') + bazel.build('//...') } stage('Bazel Tests') { bazel.test('//...') From f42aedd8fffc1908818ac4e7f00ca3c2388a42d5 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Wed, 8 Feb 2017 16:13:19 -0800 Subject: [PATCH 09/10] Do not use Jenkins bazel env --- .bazelrc.jenkins | 1 + Jenkinsfile | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.bazelrc.jenkins b/.bazelrc.jenkins index eda2c0fd2ee..664f7b13380 100644 --- a/.bazelrc.jenkins +++ b/.bazelrc.jenkins @@ -1,6 +1,7 @@ # This is from Bazel's former travis setup, to avoid blowing up the RAM usage. startup --host_jvm_args=-Xmx8192m startup --host_jvm_args=-Xms8192m +startup --batch # This is so we understand failures better build --verbose_failures diff --git a/Jenkinsfile b/Jenkinsfile index 904399033eb..93161cf53ee 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,9 @@ def bazel = new Bazel() node { gitUtils.initialize() - bazel.setVars() + // Proxy does build work correctly with Hazelcast. + // Must use .bazelrc.jenkins + bazel.setVars('', '') } mainFlow(utils) { From af85bdaca7087f9fc3d36c11469aaf935aaef0d2 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Wed, 8 Feb 2017 16:49:01 -0800 Subject: [PATCH 10/10] Set .bazelrc for postsubmit --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 93161cf53ee..db6abcaa5f4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,6 +33,7 @@ mainFlow(utils) { } if (utils.runStage('POSTSUBMIT')) { buildNode(gitUtils) { + bazel.updateBazelRc() sh 'script/release-binary' } }