[v1.x] Cherry-pick Fix mxnet-native and Docker CD pipelines (#17784)

cd/Jenkinsfile_cd_pipeline

@@ -31,7 +31,7 @@ pipeline {
     // Because each pass of the CD pipeline
     // updates Jenkins' state of the release job
     // to avoid crazy issues, we don't allow concurrent builds.
-    disableConcurrentBuilds()  
+    disableConcurrentBuilds()
   }
 
   parameters {
@@ -45,7 +45,6 @@ pipeline {
       steps {
         script {
           cd_utils = load('cd/Jenkinsfile_utils.groovy')
-
           // Update release job state in Jenkins
           cd_utils.update_release_job_state()
         }
@@ -59,7 +58,7 @@ pipeline {
 
             "Static libmxnet based release": {
               stage("Build") {
-                cd_utils.trigger_release_job("Build static libmxnet", "mxnet_lib/static", params.MXNET_VARIANTS)    
+                cd_utils.trigger_release_job("Build static libmxnet", "mxnet_lib/static", params.MXNET_VARIANTS)
               }
               stage("Releases") {
                 cd_utils.error_checked_parallel([
@@ -77,10 +76,9 @@ pipeline {
 
             "Dynamic libmxnet based release": {
               stage("Build") {
-                cd_utils.trigger_release_job("Build dynamic libmxnet", "mxnet_lib/dynamic", params.MXNET_VARIANTS)    
+                cd_utils.trigger_release_job("Build dynamic libmxnet", "mxnet_lib/dynamic", params.MXNET_VARIANTS)
               }
             }
-
           ])
         }
       }

cd/python/docker/python_images.sh

@@ -45,7 +45,7 @@ fi
 
 build() {
     # NOTE: Ensure the correct context root is passed in when building - Dockerfile expects ./wheel_build
-    docker build -t "${image_name}" --build-arg --build-arg BASE_IMAGE="${base_image}" --build-arg MXNET_COMMIT_ID=${GIT_COMMIT} -f ${resources_path}/Dockerfile ./wheel_build
+    docker build -t "${image_name}" --build-arg BASE_IMAGE="${base_image}" --build-arg MXNET_COMMIT_ID=${GIT_COMMIT} -f ${resources_path}/Dockerfile ./wheel_build
 }
 
 test() {

ci/docker/runtime_functions.sh

@@ -156,58 +156,31 @@ gather_licenses() {
 
 build_ubuntu_cpu_release() {
     set -ex
-
-    build_ccache_wrappers
-
-    make  \
-        DEV=0                         \
-        ENABLE_TESTCOVERAGE=0         \
-        USE_CPP_PACKAGE=0             \
-        USE_MKLDNN=0                  \
-        USE_BLAS=openblas             \
-        USE_SIGNAL_HANDLER=1          \
-        -j$(nproc)
+    cd /work/build
+    cmake \
+        -DUSE_MKL_IF_AVAILABLE=OFF \
+        -DUSE_MKLDNN=ON \
+        -DUSE_CUDA=OFF \
+        -G Ninja /work/mxnet
+    ninja
 }
 
-build_ubuntu_cpu_mkldnn_release() {
+build_ubuntu_cpu_native_release() {
     set -ex
-
-    build_ccache_wrappers
-
-    make  \
-        DEV=0                         \
-        ENABLE_TESTCOVERAGE=0         \
-        USE_CPP_PACKAGE=0             \
-        USE_MKLDNN=1                  \
-        USE_BLAS=openblas             \
-        USE_SIGNAL_HANDLER=1          \
-        -j$(nproc)
+    cd /work/build
+    cmake \
+        -DUSE_MKL_IF_AVAILABLE=OFF \
+        -DUSE_MKLDNN=OFF \
+        -DUSE_CUDA=OFF \
+        -G Ninja /work/mxnet
+    ninja
 }
 
 build_ubuntu_gpu_release() {
     set -ex
     # unfortunately this build has problems in 3rdparty dependencies with ccache and make
     # build_ccache_wrappers
 
-    make \
-        DEV=0                                     \
-        ENABLE_TESTCOVERAGE=0                     \
-        USE_BLAS=openblas                         \
-        USE_MKLDNN=0                              \
-        USE_CUDA=1                                \
-        USE_CUDA_PATH=/usr/local/cuda             \
-        USE_CUDNN=1                               \
-        USE_CPP_PACKAGE=0                         \
-        USE_DIST_KVSTORE=1                        \
-        USE_SIGNAL_HANDLER=1                      \
-        -j$(nproc)
-}
-
-build_ubuntu_gpu_mkldnn_release() {
-    set -ex
-    # unfortunately this build has problems in 3rdparty dependencies with ccache and make
-    # build_ccache_wrappers
-
     make \
         DEV=0                                     \
         ENABLE_TESTCOVERAGE=0                     \
@@ -224,7 +197,7 @@ build_ubuntu_gpu_mkldnn_release() {
 
 # Compiles the dynamic mxnet library
 # Parameters:
-# $1 -> mxnet_variant: the mxnet variant to build, e.g. cpu, cu100, cu92mkl, etc.
+# $1 -> mxnet_variant: the mxnet variant to build, e.g. cpu, native, cu100, cu92, etc.
 build_dynamic_libmxnet() {
     set -ex
 
@@ -235,14 +208,13 @@ build_dynamic_libmxnet() {
 
     if [[ ${mxnet_variant} = "cpu" ]]; then
         build_ubuntu_cpu_release
-    elif [[ ${mxnet_variant} = "mkl" ]]; then
-        build_ubuntu_cpu_mkldnn_release
+    elif [[ ${mxnet_variant} = "native" ]]; then
+        build_ubuntu_cpu_native_release
     elif [[ ${mxnet_variant} =~ cu[0-9]+$ ]]; then
         build_ubuntu_gpu_release
-    elif [[ ${mxnet_variant} =~ cu[0-9]+mkl$ ]]; then
-        build_ubuntu_gpu_mkldnn_release
     else
         echo "Error: Unrecognized mxnet variant '${mxnet_variant}'"
+        exit 1
     fi
 }
 

config/distribution/linux_native.cmake

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set(CMAKE_BUILD_TYPE "Distribution" CACHE STRING "Build type")
+set(CFLAGS "-mno-avx" CACHE STRING "CFLAGS")
+set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
+
+set(USE_CUDA OFF CACHE BOOL "Build with CUDA support")
+set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
+set(USE_OPENMP ON CACHE BOOL "Build with Openmp support")
+set(USE_MKL_IF_AVAILABLE OFF CACHE BOOL "Use Intel MKL if found")
+set(USE_MKLDNN OFF CACHE BOOL "Build with MKL-DNN support")
+set(USE_LAPACK ON CACHE BOOL "Build with lapack support")
+set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
+set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
+set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
+set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")