Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Commit

Permalink
[MXNET-862] Basic maven jenkins pipeline (#13450)
Browse files Browse the repository at this point in the history
* Jenkins Publish Nightly Maven

Progress

* Seperate Build, Test, and Deploy Stages with parallel
  • Loading branch information
zachgk authored and lanking520 committed Jan 8, 2019
1 parent dfc2604 commit 96439e6
Show file tree
Hide file tree
Showing 14 changed files with 457 additions and 33 deletions.
16 changes: 10 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,10 @@ else
CFLAGS += -DMXNET_USE_LIBJPEG_TURBO=0
endif

ifeq ($(CI), 1)
MAVEN_ARGS := -B
endif

# For quick compile test, used smaller subset
ALLX_DEP= $(ALL_DEP)

Expand Down Expand Up @@ -629,27 +633,27 @@ scalatestcompile:

scalapkg:
(cd $(ROOTDIR)/scala-package && \
mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
mvn package $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dcurrent_libdir="$(ROOTDIR)/lib" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")

scalaunittest:
(cd $(ROOTDIR)/scala-package && \
mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \
mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))

scalaintegrationtest:
(cd $(ROOTDIR)/scala-package && \
mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \
mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))

scalainstall:
(cd $(ROOTDIR)/scala-package && \
mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \
mvn install $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
Expand All @@ -674,14 +678,14 @@ scalarelease-perform:

scaladeploy:
(cd $(ROOTDIR)/scala-package && \
mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
mvn deploy $(MAVEN_ARGS) -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")

scaladeploylocal:
(cd $(ROOTDIR)/scala-package && \
mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
mvn deploy $(MAVEN_ARGS) -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
-DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \
-Dgpg.skip \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
Expand Down
9 changes: 5 additions & 4 deletions ci/Jenkinsfile_utils.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def init_git_win() {

// pack libraries for later use
def pack_lib(name, libs, include_gcov_data = false) {
sh """
sh returnStatus: true, script: """
set +e
echo "Packing ${libs} into ${name}"
echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
Expand All @@ -83,7 +83,7 @@ return 0
def unpack_and_init(name, libs, include_gcov_data = false) {
init_git()
unstash name
sh """
sh returnStatus: true, script: """
set +e
echo "Unpacked ${libs} from ${name}"
echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
Expand Down Expand Up @@ -147,8 +147,9 @@ def collect_test_results_windows(original_file_name, new_file_name) {
}


def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
def docker_run(platform, function_name, use_nvidia, shared_mem = '500m', env_vars = "") {
def command = "ci/build.py %ENV_VARS% --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
command = command.replaceAll('%ENV_VARS%', env_vars.length() > 0 ? "-e ${env_vars}" : '')
command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '')
command = command.replaceAll('%PLATFORM%', platform)
command = command.replaceAll('%FUNCTION_NAME%', function_name)
Expand Down
20 changes: 14 additions & 6 deletions ci/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,20 +215,21 @@ def container_run(platform: str,
local_ccache_dir: str,
command: List[str],
cleanup: Cleanup,
environment: Dict[str, str],
dry_run: bool = False) -> int:
"""Run command in a container"""
container_wait_s = 600
#
# Environment setup
#
environment = {
environment.update({
'CCACHE_MAXSIZE': '500G',
'CCACHE_TEMPDIR': '/tmp/ccache', # temp dir should be local and not shared
'CCACHE_DIR': '/work/ccache', # this path is inside the container as /work/ccache is
# mounted
'CCACHE_LOGFILE': '/tmp/ccache.log', # a container-scoped log, useful for ccache
# verification.
}
})
# These variables are passed to the container to the process tree killer can find runaway
# process inside the container
# https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller
Expand Down Expand Up @@ -446,6 +447,10 @@ def main() -> int:
parser.add_argument("--no-cache", action="store_true",
help="passes --no-cache to docker build")

parser.add_argument("-e", "--environment", nargs="*", default=[],
help="Environment variables for the docker container. "
"Specify with a list containing either names or name=value")

parser.add_argument("command",
help="command to run in the container",
nargs='*', action='append', type=str)
Expand Down Expand Up @@ -474,6 +479,9 @@ def signal_handler(signum, _):
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)

environment = dict([(e.split('=')[:2] if '=' in e else (e, os.environ[e]))
for e in args.environment])

if args.list:
print(list_platforms())
elif args.platform:
Expand All @@ -493,21 +501,21 @@ def signal_handler(signum, _):
ret = container_run(
platform=platform, nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
local_ccache_dir=args.ccache_dir, cleanup=cleanup)
local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
elif args.print_docker_run:
command = []
ret = container_run(
platform=platform, nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup)
local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup, environment=environment)
else:
# With no commands, execute a build function for the target platform
command = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
logging.info("No command specified, trying default build: %s", ' '.join(command))
ret = container_run(
platform=platform, nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
local_ccache_dir=args.ccache_dir, cleanup=cleanup)
local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)

if ret != 0:
logging.critical("Execution of %s failed with status: %d", command, ret)
Expand Down Expand Up @@ -535,7 +543,7 @@ def signal_handler(signum, _):
container_run(
platform=platform, nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
local_ccache_dir=args.ccache_dir, cleanup=cleanup)
local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
shutil.move(buildir(), plat_buildir)
logging.info("Built files left in: %s", plat_buildir)

Expand Down
3 changes: 3 additions & 0 deletions ci/docker/install/ubuntu_scala.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,7 @@ apt-get install -y openjdk-8-jre
apt-get update || true
apt-get install -y \
maven \
gnupg \
gnupg2 \
gnupg-agent \
scala
47 changes: 39 additions & 8 deletions ci/docker/runtime_functions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ clean_repo() {
git submodule update --init --recursive
}

scala_prepare() {
# Clean up maven logs
export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
}

build_ccache_wrappers() {
set -ex

Expand Down Expand Up @@ -309,7 +314,6 @@ build_amzn_linux_cpu() {
ninja -v
}


build_centos7_mkldnn() {
set -ex
cd /work/mxnet
Expand Down Expand Up @@ -842,21 +846,24 @@ unittest_ubuntu_python3_quantization_gpu() {

unittest_ubuntu_cpu_scala() {
set -ex
make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
scala_prepare
make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
}

unittest_centos7_cpu_scala() {
set -ex
cd /work/mxnet
scala_prepare
make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
}

unittest_ubuntu_cpu_clojure() {
set -ex
make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
scala_prepare
make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
./contrib/clojure-package/ci-test.sh
}

Expand Down Expand Up @@ -1000,8 +1007,9 @@ integrationtest_ubuntu_cpu_dist_kvstore() {

integrationtest_ubuntu_gpu_scala() {
set -ex
make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
scala_prepare
make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 CI=1
make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
}

integrationtest_ubuntu_gpu_dist_kvstore() {
Expand Down Expand Up @@ -1065,7 +1073,6 @@ build_docs() {
popd
}


# Functions that run the nightly Tests:

#Runs Apache RAT Check on MXNet Source for License Headers
Expand Down Expand Up @@ -1263,6 +1270,30 @@ deploy_jl_docs() {
# ...
}

publish_scala_build() {
set -ex
pushd .
scala_prepare
./scala-package/dev/build.sh
popd
}

publish_scala_test() {
set -ex
pushd .
scala_prepare
./scala-package/dev/test.sh
popd
}

publish_scala_deploy() {
set -ex
pushd .
scala_prepare
./scala-package/dev/deploy.sh
popd
}

# broken_link_checker

broken_link_checker() {
Expand Down
3 changes: 1 addition & 2 deletions ci/docker_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,8 +210,7 @@ def _get_dockerhub_credentials(): # pragma: no cover
logging.exception("The request was invalid due to:")
elif client_error.response['Error']['Code'] == 'InvalidParameterException':
logging.exception("The request had invalid params:")
else:
raise
raise
else:
secret = get_secret_value_response['SecretString']
secret_dict = json.loads(secret)
Expand Down
105 changes: 105 additions & 0 deletions ci/publish/Jenkinsfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
// -*- mode: groovy -*-

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

// Jenkins pipeline
// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/

//mxnet libraries
mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/test-classes/**, scala-package/local-snapshot/**'

// timeout in minutes
max_time = 120

node('restricted-utility') {
// Loading the utilities requires a node context unfortunately
checkout scm
utils = load('ci/Jenkinsfile_utils.groovy')
}
utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')

// CPU and GPU. OSX nodes are not currently supported by Jenkins
def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU]
def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu']

def wrapStep(nodeToRun, workspaceName, step) {
return {
node(nodeToRun) {
ws("workspace/${workspaceName}") {
timeout(time: max_time, unit: 'MINUTES') {
step()
}
}
}
}
}

def toBuild = [:]
def labels = ['cpu'] // , 'gpu']
for (x in labels) {
def label = x // Required due to language
toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") {
env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label]
utils.init_git()
utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE')
utils.pack_lib("scala_${label}", mx_scala_pub, false)
}
}

def toTest = [:]
def systems = ['ubuntu'] // , 'centos7']
for (x in labels) {
def label = x // Required due to language
for (y in systems) {
def system = y // Required due to language
toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") {
utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
utils.docker_run("${system}_${label}", 'publish_scala_test', label == 'gpu')
}
}
}

def toDeploy = [:]
for (x in labels) {
def label = x // Required due to language
toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") {
env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label]
utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION')
}
}

utils.main_wrapper(
core_logic: {
stage('Build Packages') {
parallel toBuild
}
stage('Test Packages') {
parallel toTest
}
stage('Deploy Packages') {
parallel toDeploy
}
}
,
failure_handler: {
if (currentBuild.result == "FAILURE") {
// emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
}
}
)
2 changes: 1 addition & 1 deletion scala-package/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ core/src/main/scala/org/apache/mxnet/NDArrayRandomAPIBase.scala
core/src/main/scala/org/apache/mxnet/SymbolRandomAPIBase.scala
examples/scripts/infer/images/
examples/scripts/infer/models/
local-snapshot
local-snapshot
Loading

0 comments on commit 96439e6

Please sign in to comment.