Linux compile on Debian 6.

We used to compile on a recent ubuntu version, which is needed for
cross-compiling to Windows and OS X. This poses the problem that it links
against a relatively recent version of libc, so the resulting binary cannot work
on older linux distributions. To work around this issue we tried to compile
statically linked binaries. Unfortunately, statically linking against libc
breaks in part the name resolution, in particular resolving the names from
/etc/hosts.

This pull requests makes 2 Docker images: one based on ubuntu for cross
compiling to os x and windows and one based on debian 6 for compiling for Linux
(both 32 and 64 bits versions). Libpcap continues to be statically linked for
Linux.

Because the Dockerfiles have drifted quite a bit from the versions from XGO, I
moved them to this repository.

This closes elastic/beats#225.

Makefile

@@ -9,16 +9,28 @@ all: packetbeat/deb packetbeat/rpm packetbeat/darwin packetbeat/win packetbeat/b
 
 .PHONY: packetbeat topbeat
 packetbeat topbeat: xgo-image build
-	cd build && xgo -image=tudorg/beats-builder -static \
+	cd build && xgo -image=tudorg/beats-builder \
 		-before-build=../xgo-scripts/$@_before_build.sh \
 		-branch $(RELEASE) \
 		github.com/elastic/$@
 
-%/deb: % build/god-linux-386 build/god-linux-amd64 fpm-image
+.PHONY: packetbeat-linux topbeat-linux
+packetbeat-linux: xgo-image build
+	cd build && xgo -image=tudorg/beats-builder-deb6 \
+		-before-build=../xgo-scripts/packetbeat_before_build.sh \
+		-branch $(RELEASE) \
+		github.com/elastic/packetbeat
+topbeat-linux: xgo-image build
+	cd build && xgo -image=tudorg/beats-builder-deb6 \
+		-before-build=../xgo-scripts/topbeat_before_build.sh \
+		-branch $(RELEASE) \
+		github.com/elastic/topbeat
+
+%/deb: %-linux build/god-linux-386 build/god-linux-amd64 fpm-image
 	ARCH=386 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/debian/build.sh
 	ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/debian/build.sh
 
-%/rpm: % build/god-linux-386 build/god-linux-amd64 fpm-image
+%/rpm: %-linux build/god-linux-386 build/god-linux-amd64 fpm-image
 	ARCH=386 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/centos/build.sh
 	ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/centos/build.sh
 
@@ -28,7 +40,7 @@ packetbeat topbeat: xgo-image build
 %/win: % fpm-image
 	ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/windows/build.sh
 
-%/bin: % fpm-image
+%/bin: %-linux fpm-image
 	ARCH=386 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/binary/build.sh
 	ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/binary/build.sh
 
@@ -39,7 +51,8 @@ deps:
 
 .PHONY: xgo-image
 xgo-image:
-	docker build -t tudorg/beats-builder docker/xgo-image/
+	cd docker/xgo-image/; ./build.sh
+	cd docker/xgo-image-deb6/; ./build.sh
 
 .PHONY: fpm-image
 fpm-image:
@@ -60,11 +73,15 @@ s3-nightlies-upload: all
 	echo $(BUILDID) > build/upload/build_id.txt
 	aws s3 cp --recursive --acl public-read build/upload s3://beats-nightlies
 
+.PHONY: release-upload
+release-upload:
+	aws s3 cp --recursive --acl public-read build/upload s3://download.elasticsearch.org/beats/
+
 .PHONY: run-interactive
 run-interactive:
 	docker run -t -i -v $(shell pwd)/build:/build \
 		-v $(shell pwd)/xgo-scripts/:/scripts \
-		--entrypoint=bash tudorg/beats-builder
+		--entrypoint=bash tudorg/beats-builder-deb6
 .PHONY: clean
 clean:
 	rm -rf build/ || true

docker/xgo-image-deb6/base/Dockerfile

@@ -0,0 +1,44 @@
+# Go cross compiler (xgo): Base cross-compilation layer
+# Copyright (c) 2014 Péter Szilágyi. All rights reserved.
+#
+# Released under the MIT license.
+
+FROM debian:6
+
+MAINTAINER Péter Szilágyi <peterke@gmail.com>
+
+# Configure the Go environment, since it's not going to change
+ENV PATH   /usr/local/go/bin:$PATH
+ENV GOPATH /go
+
+
+# Inject the remote file fetcher and checksum verifier
+ADD fetch.sh /fetch.sh
+ENV FETCH /fetch.sh
+RUN chmod +x $FETCH
+
+
+# Make sure apt-get is up to date and dependent packages are installed
+RUN \
+  apt-get update && \
+  apt-get install -y automake autogen build-essential ca-certificates \
+    gcc-multilib \
+    clang llvm-dev  libtool libxml2-dev uuid-dev libssl-dev pkg-config \
+    patch make xz-utils cpio wget unzip git mercurial bzr --no-install-recommends
+
+# Inject the Go package downloader and tool-chain bootstrapper
+ADD bootstrap.sh /bootstrap.sh
+ENV BOOTSTRAP /bootstrap.sh
+RUN chmod +x $BOOTSTRAP
+
+# Inject the C dependency cross compiler
+ADD build_deps.sh /build_deps.sh
+ENV BUILD_DEPS /build_deps.sh
+RUN chmod +x $BUILD_DEPS
+
+# Inject the container entry point, the build script
+ADD build.sh /build.sh
+ENV BUILD /build.sh
+RUN chmod +x $BUILD
+
+ENTRYPOINT ["/build.sh"]

docker/xgo-image-deb6/base/bootstrap.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Contains the Go tool-chain bootstrapper, that retrieves all the configured
+# distribution packages, extracts the binaries and deletes anything not needed.
+#
+# Usage: bootstrap.sh
+#
+# Needed environment variables:
+#   FETCH - Remote file fetcher and checksum verifier (injected by image)
+#   DIST_LINUX_64,  DIST_LINUX_64_SHA1  - 64 bit Linux Go binaries and checksum
+#   DIST_LINUX_32,  DIST_LINUX_32_SHA1  - 32 bit Linux Go binaries and checksum
+#   DIST_LINUX_ARM, DIST_LINUX_ARM_SHA1 - ARM v5 Linux Go binaries and checksum
+#   DIST_OSX_64,    DIST_OSX_64_SHA1    - 64 bit Mac OSX Go binaries and checksum
+#   DIST_OSX_32,    DIST_OSX_32_SHA1    - 32 bit Mac OSX Go binaries and checksum
+#   DIST_WIN_64,    DIST_WIN_64_SHA1    - 64 bit Windows Go binaries and checksum
+#   DIST_WIN_32,    DIST_WIN_32_SHA1    - 32 bit Windows Go binaries and checksum 
+set -e
+
+# Download and verify all the binary packages
+$FETCH $DIST_LINUX_64  $DIST_LINUX_64_SHA1
+$FETCH $DIST_LINUX_32  $DIST_LINUX_32_SHA1
+
+# Extract the 64 bit Linux package as the primary Go SDK
+tar -C /usr/local -xzf `basename $DIST_LINUX_64`
+
+# Extract all other packages as secondary ones, keeping only the binaries
+tar -C /usr/local --wildcards -xzf `basename $DIST_LINUX_32` go/pkg/linux_386*
+GOOS=linux GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap
+
+# Delete all the intermediate downloaded files
+rm -f `basename $DIST_LINUX_64` `basename $DIST_LINUX_32`

docker/xgo-image-deb6/base/build.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+#
+# Contains the main cross compiler, that individually sets up each target build
+# platform, compiles all the C dependencies, then build the requested executable
+# itself.
+#
+# Usage: build.sh <import path>
+#
+# Needed environment variables:
+#   REPO_REMOTE - Optional VCS remote if not the primary repository is needed
+#   REPO_BRANCH - Optional VCS branch to use, if not the master branch
+#   DEPS        - Optional list of C dependency packages to build
+#   PACK        - Optional sub-package, if not the import path is being built
+#   OUT         - Optional output prefix to override the package name
+#   FLAG_V      - Optional verbosity flag to set on the Go builder
+#   FLAG_RACE   - Optional race flag to set on the Go builder
+
+# Download the canonical import path (may fail, don't allow failures beyond)
+echo "Fetching main repository $1..."
+mkdir -p $GOPATH/src/`dirname $1`
+cd $GOPATH/src/`dirname $1`
+git clone https://$1.git
+
+set -e
+
+cd $GOPATH/src/$1
+export GOPATH=$GOPATH:`pwd`/Godeps/_workspace
+
+# Switch over the code-base to another checkout if requested
+if [ "$REPO_REMOTE" != "" ]; then
+  echo "Switching over to remote $REPO_REMOTE..."
+  if [ -d ".git" ]; then
+    git remote set-url origin $REPO_REMOTE
+    git pull
+  elif [ -d ".hg" ]; then
+    echo -e "[paths]\ndefault = $REPO_REMOTE\n" >> .hg/hgrc
+    hg pull
+  fi
+fi
+
+if [ "$REPO_BRANCH" != "" ]; then
+  echo "Switching over to branch $REPO_BRANCH..."
+  if [ -d ".git" ]; then
+    git checkout $REPO_BRANCH
+  elif [ -d ".hg" ]; then
+    hg checkout $REPO_BRANCH
+  fi
+fi
+
+# Download all the C dependencies
+echo "Fetching dependencies..."
+mkdir -p /deps
+DEPS=($DEPS) && for dep in "${DEPS[@]}"; do
+  echo Downloading $dep
+  if [ "${dep##*.}" == "tar" ]; then wget -q $dep -O - | tar -C /deps -x; fi
+  if [ "${dep##*.}" == "gz" ]; then wget -q $dep -O - | tar -C /deps -xz; fi
+  if [ "${dep##*.}" == "bz2" ]; then wget -q $dep -O - | tar -C /deps -xj; fi
+done
+
+# Configure some global build parameters
+NAME=`basename $1/$PACK`
+if [ "$OUT" != "" ]; then
+  NAME=$OUT
+fi
+
+
+if [ "$FLAG_V" == "true" ]; then V=-v; fi
+if [ "$FLAG_RACE" == "true" ]; then R=-race; fi
+if [ "$STATIC" == "true" ]; then LDARGS=--ldflags\ \'-extldflags\ \"-static\"\'; fi
+
+if [ -n $BEFORE_BUILD ]; then
+	chmod +x /scripts/$BEFORE_BUILD
+	echo "Execute /scripts/$BEFORE_BUILD"
+	/scripts/$BEFORE_BUILD
+fi
+
+# Build for each platform individually
+echo "Compiling for linux/amd64..."
+HOST=x86_64-linux PREFIX=/usr/local $BUILD_DEPS /deps
+GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK
+sh -c "GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build $V $R $LDARGS -o $NAME-linux-amd64$R ./$PACK"
+
+echo "Compiling for linux/386..."
+HOST=i686-linux PREFIX=/usr/local $BUILD_DEPS /deps
+GOOS=linux GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK
+sh -c "GOOS=linux GOARCH=386 CGO_ENABLED=1 go build $V $LDARGS -o $NAME-linux-386 ./$PACK"
+
+#echo "Compiling for linux/arm..."
+#CC=arm-linux-gnueabi-gcc HOST=arm-linux PREFIX=/usr/local/arm $BUILD_DEPS /deps
+#CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go get -d ./$PACK
+#CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go build $V -o $NAME-linux-arm ./$PACK
+
+#echo "Compiling for windows/amd64..."
+#CC=x86_64-w64-mingw32-gcc HOST=x86_64-w64-mingw32 PREFIX=/usr/x86_64-w64-mingw32 $BUILD_DEPS /deps
+#CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK
+#CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go build  $V $R -o $NAME-windows-amd64$R.exe ./$PACK
+
+#echo "Compiling for windows/386..."
+#CC=i686-w64-mingw32-gcc HOST=i686-w64-mingw32 PREFIX=/usr/i686-w64-mingw32 $BUILD_DEPS /deps
+#CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK
+#CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-windows-386.exe ./$PACK
+
+#echo "Compiling for darwin/amd64..."
+#CC=o64-clang HOST=x86_64-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS /deps
+#CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK
+#CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-darwin-amd64$R ./$PACK
+
+#echo "Compiling for darwin/386..."
+#CC=o32-clang HOST=i386-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS /deps
+#CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK
+#CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-darwin-386 ./$PACK
+
+echo "Moving binaries to host..."
+cp `ls -t | head -n 2` /build

docker/xgo-image-deb6/base/build_deps.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Contains the a dependency builder to iterate over all installed dependencies
+# and cross compile them to the requested target platform.
+# 
+# Usage: build_deps.sh <dependency folder>
+#
+# Needed environment variables:
+#   CC     - C cross compiler to use for the build
+#   HOST   - Target platform to build (used to find the needed tool-chains)
+#   PREFIX - File-system path where to install the built binaries
+set -e
+
+# Remove any previous build leftovers, and copy a fresh working set (clean doesn't work for cross compiling)
+rm -rf /deps-build && cp -r $1 /deps-build
+
+# Build all the dependencies (no order for now)
+for dep in `ls /deps-build`; do
+	echo "Configuring dependency $dep for $HOST..."
+	(cd /deps-build/$dep && ./configure --disable-shared --host=$HOST --prefix=$PREFIX --silent)
+
+	echo "Building dependency $dep for $HOST..."
+	(cd /deps-build/$dep && make --silent -j install)
+done
+
+# Remove any build artifacts
+rm -rf /deps-build

docker/xgo-image-deb6/base/fetch.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Contains a simple fetcher to download a file from a remote URL and verify its
+# SHA1 checksum.
+#
+# Usage: fetch.sh <remote URL> <SHA1 checksum>
+set -e
+
+# Pull the file from the remote URL
+file=`basename $1`
+echo "Downloading $1..."
+wget --no-check-certificate -q $1
+
+# Generate a desired checksum report and check against it
+echo "$2  $file" > $file.sum
+sha1sum -c $file.sum
+rm $file.sum

docker/xgo-image-deb6/beats-builder/Dockerfile

@@ -0,0 +1,18 @@
+FROM tudorg/xgo-deb6-1.4.2
+
+MAINTAINER Tudor Golubenco <tudor@elastic.co>
+
+# Get libpcap binaries for linux
+RUN \
+	mkdir -p /libpcap && \
+    wget http://ftp.de.debian.org/debian/pool/main/libp/libpcap/libpcap0.8-dev_1.1.1-2+squeeze1_i386.deb && \
+	dpkg -x libpcap0.8-dev_*_i386.deb /libpcap/i386 && \
+	wget http://ftp.de.debian.org/debian/pool/main/libp/libpcap/libpcap0.8-dev_1.1.1-2+squeeze1_amd64.deb && \
+	dpkg -x libpcap0.8-dev_*_amd64.deb /libpcap/amd64 && \
+	rm libpcap0.8-dev*.deb
+RUN \
+	apt-get update && \
+	apt-get install -y libpcap0.8-dev
+
+# add patch for gopacket
+ADD gopacket_pcap.patch /gopacket_pcap.patch

docker/xgo-image-deb6/beats-builder/gopacket_pcap.patch

@@ -0,0 +1,22 @@
+diff --git a/Godeps/_workspace/src/github.com/tsg/gopacket/pcap/pcap.go b/Godeps/_workspace/src/github.
+index d2465bb..7b23b84 100644
+--- a/Godeps/_workspace/src/github.com/tsg/gopacket/pcap/pcap.go
++++ b/Godeps/_workspace/src/github.com/tsg/gopacket/pcap/pcap.go
+@@ -8,12 +8,13 @@
+ package pcap
+
+ /*
+-#cgo linux LDFLAGS: -lpcap
++#cgo linux,386 LDFLAGS: /libpcap/i386/usr/lib/libpcap.a
++#cgo linux,amd64 LDFLAGS: /libpcap/amd64/usr/lib/libpcap.a
+ #cgo freebsd LDFLAGS: -lpcap
+ #cgo darwin LDFLAGS: -lpcap
+-#cgo windows CFLAGS: -I C:/WpdPack/Include
+-#cgo windows,386 LDFLAGS: -L C:/WpdPack/Lib -lwpcap
+-#cgo windows,amd64 LDFLAGS: -L C:/WpdPack/Lib/x64 -lwpcap
++#cgo windows CFLAGS: -I /libpcap/win/WpdPack/Include
++#cgo windows,386 LDFLAGS: -L /libpcap/win/WpdPack/Lib -lwpcap
++#cgo windows,amd64 LDFLAGS: -L /libpcap/win/WpdPack/Lib/x64 -lwpcap
+ #include <stdlib.h>
+ #include <pcap.h>
+

docker/xgo-image-deb6/build.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+docker build -t tudorg/xgo-deb6-base base/ && \
+    docker build -t tudorg/xgo-deb6-1.4.2 go-1.4.2/ &&
+    docker build -t tudorg/beats-builder-deb6 beats-builder

docker/xgo-image-deb6/go-1.4.2/Dockerfile

@@ -0,0 +1,18 @@
+# Go cross compiler (xgo): Go 1.4.2 layer
+# Copyright (c) 2014 Péter Szilágyi. All rights reserved.
+#
+# Released under the MIT license.
+
+FROM tudorg/xgo-deb6-base
+
+MAINTAINER Péter Szilágyi <peterke@gmail.com>
+
+# Configure the Go packages and bootstrap them
+RUN \
+  export DIST_LINUX_64=https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz && \
+  export DIST_LINUX_64_SHA1=5020af94b52b65cc9b6f11d50a67e4bae07b0aff && \
+  \
+  export DIST_LINUX_32=https://storage.googleapis.com/golang/go1.4.2.linux-386.tar.gz && \
+  export DIST_LINUX_32_SHA1=50557248e89b6e38d395fda93b2f96b2b860a26a && \
+  \
+  $BOOTSTRAP