ORC-564: Change the docker files to support local builds.

Fixes #444 Signed-off-by: Owen O'Malley <omalley@apache.org>
apache · Nov 1, 2019 · 953c00f · 953c00f
1 parent db1fc71
commit 953c00f
Show file tree

Hide file tree

Showing 12 changed files with 110 additions and 73 deletions.
diff --git a/docker/README b/docker/README
@@ -1,14 +1,25 @@
-These scripts are useful for testing on different versions of Linux
-assuming that you are running a version of Linux that has docker
-available.
+These scripts are useful for building and testing on different versions of
+Linux.
+
+ORC is currently supported on:
+* CentOS 6, 7, and 8
+* Debian 8, 9, and 10
+* Ubuntu 14, 16, and 18
 
 To test against all of the Linux OSes against Apache's master branch:
-1. ./run-all.sh apache master
+1. cd docker
+2. ./run-all.sh apache master
+
+Using 'local' as the owner will cause the scripts to use the local repository.
+
+The scripts are:
+1. run-all.sh *owner* *branch* - test the given owner's branch on all OSes
+2. run-one.sh *owner* *branch* *os* - test the owner's branch on one OS
+3. reinit.sh - rebuild all of the base images without the image cache
 
-To run the docker scripts:
-1. cd $os
+A base image for each OS is built using:
+1. cd docker/$os
 2. docker build -t orc-$os .
-3. docker run orc-$os
 
 To clean up docker:
 1. docker container prune

diff --git a/docker/centos6/Dockerfile b/docker/centos6/Dockerfile
@@ -62,8 +62,10 @@ RUN cd snappy-1.1.3 && \
 
 VOLUME /root/.m2/repository
 
-CMD git clone https://github.com/apache/orc.git -b master && \
-  mkdir orc/build && \
-  cd orc/build && \
-  cmake .. && \ 
-  make package test-out
+CMD if [ ! -d orc ]; then \
+      git clone https://github.com/apache/orc.git -b master; \
+    fi && \
+    mkdir build && \
+    cd build && \
+    cmake -DSNAPPY_HOME=/usr/local -DPROTOBUF_HOME=/usr/local ../orc && \
+    make package test-out
diff --git a/docker/centos7/Dockerfile b/docker/centos7/Dockerfile
@@ -54,8 +54,10 @@ RUN cd snappy-1.1.3 && \
 
 VOLUME /root/.m2/repository
 
-CMD git clone https://github.com/apache/orc.git -b master && \
-  mkdir orc/build && \
-  cd orc/build && \
-  cmake .. && \ 
-  make package test-out
+CMD if [ ! -d orc ]; then \
+      git clone https://github.com/apache/orc.git -b master; \
+    fi && \
+    mkdir build && \
+    cd build && \
+    cmake -DSNAPPY_HOME=/usr/local ../orc && \
+    make package test-out
diff --git a/docker/centos8/Dockerfile b/docker/centos8/Dockerfile
@@ -46,8 +46,10 @@ WORKDIR /root
 
 VOLUME /root/.m2/repository
 
-CMD git clone https://github.com/apache/orc.git -b master && \
-  mkdir orc/build && \
-  cd orc/build && \
-  cmake .. && \ 
-  make package test-out
+CMD if [ ! -d orc ]; then \
+      git clone https://github.com/apache/orc.git -b master; \
+    fi && \
+    mkdir build && \
+    cd build && \
+    cmake ../orc && \
+    make package test-out
diff --git a/docker/debian10/Dockerfile b/docker/debian10/Dockerfile
@@ -36,8 +36,10 @@ WORKDIR /root
 
 VOLUME /root/.m2/repository
 
-CMD git clone https://github.com/apache/orc.git -b master && \
-  mkdir orc/build && \
-  cd orc/build && \
-  cmake .. && \
-  make package test-out
+CMD if [ ! -d orc ]; then \
+      git clone https://github.com/apache/orc.git -b master; \
+    fi && \
+    mkdir build && \
+    cd build && \
+    cmake ../orc && \
+    make package test-out
diff --git a/docker/debian8/Dockerfile b/docker/debian8/Dockerfile
@@ -53,8 +53,10 @@ RUN cd snappy-1.1.3 && \
 
 VOLUME /root/.m2/repository
 
-CMD git clone https://github.com/apache/orc.git -b master && \
-  mkdir orc/build && \
-  cd orc/build && \
-  cmake .. && \
-  make package test-out
+CMD if [ ! -d orc ]; then \
+      git clone https://github.com/apache/orc.git -b master; \
+    fi && \
+    mkdir build && \
+    cd build && \
+    cmake -DSNAPPY_HOME=/usr/local ../orc && \
+    make package test-out
diff --git a/docker/debian9/Dockerfile b/docker/debian9/Dockerfile
@@ -36,8 +36,10 @@ WORKDIR /root
 
 VOLUME /root/.m2/repository
 
-CMD git clone https://github.com/apache/orc.git -b master && \
-  mkdir orc/build && \
-  cd orc/build && \
-  cmake .. && \
-  make package test-out
+CMD if [ ! -d orc ]; then \
+      git clone https://github.com/apache/orc.git -b master; \
+    fi && \
+    mkdir build && \
+    cd build && \
+    cmake ../orc && \
+    make package test-out
diff --git a/docker/run-one.sh b/docker/run-one.sh
@@ -25,24 +25,30 @@ function failure {
     exit 1
 }
 
-CLONE="git clone $URL -b $BRANCH"
-MAKEDIR="mkdir orc/build && cd orc/build"
 VOLUME="--volume m2cache:/root/.m2/repository"
+if [ $GITHUB_USER == "local" ]; then
+  BRANCH=`git status| head -1 | sed -e 's/On branch //'`
+  echo "Started local run for $BRANCH on $OS at $(date)"
+  docker run $VOLUME -v`pwd`/..:/root/orc "orc-$OS" || failure
+else
+  CLONE="git clone $URL -b $BRANCH"
+  MAKEDIR="mkdir orc/build && cd orc/build"
 
-echo "Started $GITHUB_USER/$BRANCH on $OS at $(date)"
+  echo "Started $GITHUB_USER/$BRANCH on $OS at $(date)"
 
-case $OS in
-centos6|ubuntu12)
-   OPTS="-DSNAPPY_HOME=/usr/local -DPROTOBUF_HOME=/usr/local"
-   ;;
-centos7|debian8|ubuntu14)
-   OPTS="-DSNAPPY_HOME=/usr/local"
-   ;;
-*)
-   OPTS=""
-   ;;
-esac
-docker run $VOLUME "orc-$OS" /bin/bash -c \
+  case $OS in
+  centos6|ubuntu12)
+     OPTS="-DSNAPPY_HOME=/usr/local -DPROTOBUF_HOME=/usr/local"
+     ;;
+  centos7|debian8|ubuntu14)
+     OPTS="-DSNAPPY_HOME=/usr/local"
+     ;;
+  *)
+     OPTS=""
+     ;;
+  esac
+  docker run $VOLUME "orc-$OS" /bin/bash -c \
 	 "$CLONE && $MAKEDIR && cmake $OPTS .. && make package test-out" \
-         || failure
+      || failure
+fi
 echo "Finished $OS at $(date)"
diff --git a/docker/ubuntu14/Dockerfile b/docker/ubuntu14/Dockerfile
@@ -51,8 +51,10 @@ RUN cd snappy-1.1.3 && \
 
 VOLUME /root/.m2/repository
 
-CMD git clone https://github.com/apache/orc.git -b master && \
-  mkdir orc/build && \
-  cd orc/build && \
-  cmake .. && \ 
-  make package test-out
+CMD if [ ! -d orc ]; then \
+      git clone https://github.com/apache/orc.git -b master; \
+    fi && \
+    mkdir build && \
+    cd build && \
+    cmake -DSNAPPY_HOME=/usr/local ../orc && \
+    make package test-out
diff --git a/docker/ubuntu16-clang5/Dockerfile b/docker/ubuntu16-clang5/Dockerfile
@@ -42,8 +42,10 @@ ENV CXX=clang++-5.0
 WORKDIR /root
 VOLUME /root/.m2/repository
 
-CMD git clone https://github.com/apache/orc.git -b master && \
-  mkdir orc/build && \
-  cd orc/build && \
-  cmake .. && \
-  make package test-out
+CMD if [ ! -d orc ]; then \
+      git clone https://github.com/apache/orc.git -b master; \
+    fi && \
+    mkdir build && \
+    cd build && \
+    cmake ../orc && \
+    make package test-out
diff --git a/docker/ubuntu16/Dockerfile b/docker/ubuntu16/Dockerfile
@@ -36,8 +36,10 @@ RUN apt-get install -y \
 WORKDIR /root
 VOLUME /root/.m2/repository
 
-CMD git clone https://github.com/apache/orc.git -b master && \
-  mkdir orc/build && \
-  cd orc/build && \
-  cmake .. && \
-  make package test-out
+CMD if [ ! -d orc ]; then \
+      git clone https://github.com/apache/orc.git -b master; \
+    fi && \
+    mkdir build && \
+    cd build && \
+    cmake ../orc && \
+    make package test-out
diff --git a/docker/ubuntu18/Dockerfile b/docker/ubuntu18/Dockerfile
@@ -38,8 +38,10 @@ RUN update-java-alternatives --set java-1.8.0-openjdk-amd64
 WORKDIR /root
 VOLUME /root/.m2/repository
 
-CMD git clone https://github.com/apache/orc.git -b master && \
-  mkdir orc/build && \
-  cd orc/build && \
-  cmake .. && \
-  make package test-out
+CMD if [ ! -d orc ]; then \
+      git clone https://github.com/apache/orc.git -b master; \
+    fi && \
+    mkdir build && \
+    cd build && \
+    cmake ../orc && \
+    make package test-out