Merge pull request #9370 from gyuho/hack

*: fix Makefile, move GOPATH setup to build script

build

@@ -1,11 +1,11 @@
-#!/bin/sh -e
+#!/usr/bin/env bash
 
 # set some environment variables
 ORG_PATH="github.com/coreos"
 REPO_PATH="${ORG_PATH}/etcd"
 
 GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound")
-if [ ! -z "$FAILPOINTS" ]; then
+if [[ ! -z "$FAILPOINTS" ]]; then
 	GIT_SHA="$GIT_SHA"-FAILPOINTS
 fi
 
@@ -17,7 +17,7 @@ toggle_failpoints() {
 	mode="$1"
 	if which gofail >/dev/null 2>&1; then
 		gofail "$mode" etcdserver/ internal/mvcc/backend/
-	elif [ "$mode" != "disable" ]; then
+	elif [[ "$mode" != "disable" ]]; then
 		echo "FAILPOINTS set but gofail not found"
 		exit 1
 	fi
@@ -26,44 +26,47 @@ toggle_failpoints() {
 etcd_setup_gopath() {
 	echo "Setting GOPATH from vendor directory at 'gopath'"
 	d=$(dirname "$0")
-	CDIR=$(cd "$d" && pwd)
-	cd "$CDIR"
+	CDIR=$(cd "$d" || return && pwd)
+	cd "$CDIR" || return
 	etcdGOPATH="${CDIR}/gopath"
 	# preserve old gopath to support building with unvendored tooling deps (e.g., gofail)
-	if [ -n "$GOPATH" ]; then
+	if [[ -n "$GOPATH" ]]; then
 		GOPATH=":$GOPATH"
 	fi
 	rm -rf "${etcdGOPATH:?}/"
 	mkdir -p "${etcdGOPATH}/vendor" "${etcdGOPATH}/etcd_src/src/github.com/coreos"
 	export GOPATH=${etcdGOPATH}/vendor:${etcdGOPATH}/etcd_src${GOPATH}
 	ln -s "${CDIR}/vendor" "${etcdGOPATH}/vendor/src"
 	ln -s "${CDIR}" "${etcdGOPATH}/etcd_src/src/github.com/coreos/etcd"
-
-	#ln -s "${CDIR}/vendor" "${etcdGOPATH}/src"
-	#ln -s "${CDIR}" "${etcdGOPATH}/src/github.com/coreos"
 }
 
 toggle_failpoints_default() {
 	mode="disable"
-	if [ ! -z "$FAILPOINTS" ]; then mode="enable"; fi
+	if [[ ! -z "$FAILPOINTS" ]]; then mode="enable"; fi
 	toggle_failpoints "$mode"
 }
 
 etcd_build() {
 	out="bin"
-	if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
+	if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
 	toggle_failpoints_default
-	# Static compilation is useful when etcd is run in a container. $GO_BUILD_FLAGS is OK
 
+	# Static compilation is useful when etcd is run in a container. $GO_BUILD_FLAGS is OK
 	# shellcheck disable=SC2086
-	CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "$GO_LDFLAGS" -o "${out}/etcd" ${REPO_PATH} || return
+	CGO_ENABLED=0 go build $GO_BUILD_FLAGS \
+		-installsuffix cgo \
+		-ldflags "$GO_LDFLAGS" \
+		-o "${out}/etcd" ${REPO_PATH} || return
 	# shellcheck disable=SC2086
-	CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "$GO_LDFLAGS" -o "${out}/etcdctl" ${REPO_PATH}/etcdctl || return
+	CGO_ENABLED=0 go build $GO_BUILD_FLAGS \
+		-installsuffix cgo \
+		-ldflags "$GO_LDFLAGS" \
+		-o "${out}/etcdctl" ${REPO_PATH}/etcdctl || return
 }
 
 tools_build() {
 	out="bin"
-	if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
+	if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
 	tools_path="benchmark
 	etcd-dump-db
 	etcd-dump-logs
@@ -84,6 +87,10 @@ tools_build() {
 
 toggle_failpoints_default
 
+if [[ "${ETCD_SETUP_GOPATH}" == "1" ]]; then
+	etcd_setup_gopath
+fi
+
 # only build when called directly, not sourced
 if echo "$0" | grep "build$" >/dev/null; then
 	etcd_build

hack/scripts-dev/Makefile

@@ -71,14 +71,23 @@ pull-docker-test:
 # Example:
 #   make build-docker-test -f ./hack/scripts-dev/Makefile
 #   make compile-with-docker-test -f ./hack/scripts-dev/Makefile
+#   make compile-setup-gopath-with-docker-test -f ./hack/scripts-dev/Makefile
 
 compile-with-docker-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	docker run \
+	  --rm \
+	  --mount type=bind,source=`pwd`,destination=/go/src/github.com/coreos/etcd \
+	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
+	  /bin/bash -c "GO_BUILD_FLAGS=-v ./build && ./bin/etcd --version"
+
+compile-setup-gopath-with-docker-test:
 	$(info GO_VERSION: $(GO_VERSION))
 	docker run \
 	  --rm \
 	  --mount type=bind,source=`pwd`,destination=/etcd \
 	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
-	  /bin/bash -c "cd /etcd && GO_BUILD_FLAGS=-v ./build && ./bin/etcd --version"
+	  /bin/bash -c "cd /etcd && ETCD_SETUP_GOPATH=1 GO_BUILD_FLAGS=-v ./build && ./bin/etcd --version && rm -rf ./gopath"
 
 
 
@@ -234,6 +243,7 @@ docker-static-ip-test-certs-metrics-proxy-run:
 #   make push-docker-dns-test -f ./hack/scripts-dev/Makefile
 #   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
 #   make pull-docker-dns-test -f ./hack/scripts-dev/Makefile
+#   make docker-dns-test-insecure-run -f ./hack/scripts-dev/Makefile
 #   make docker-dns-test-certs-run -f ./hack/scripts-dev/Makefile
 #   make docker-dns-test-certs-gateway-run -f ./hack/scripts-dev/Makefile
 #   make docker-dns-test-certs-wildcard-run -f ./hack/scripts-dev/Makefile
@@ -263,6 +273,20 @@ pull-docker-dns-test:
 	$(info GO_VERSION: $(GO_VERSION))
 	docker pull gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION)
 
+docker-dns-test-insecure-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/insecure,destination=/insecure \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /insecure/run.sh && rm -rf m*.etcd"
+
 docker-dns-test-certs-run:
 	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))

hack/scripts-dev/docker-dns/insecure/Procfile

@@ -0,0 +1,6 @@
+# Use goreman to run `go get github.com/mattn/goreman`
+etcd1: ./etcd --name m1 --data-dir /tmp/m1.data --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://m1.etcd.local:2379 --listen-peer-urls http://127.0.0.1:2380 --initial-advertise-peer-urls=http://m1.etcd.local:2380 --initial-cluster-token tkn --initial-cluster=m1=http://m1.etcd.local:2380,m2=http://m2.etcd.local:22380,m3=http://m3.etcd.local:32380
+
+etcd2: ./etcd --name m2 --data-dir /tmp/m2.data --listen-client-urls http://127.0.0.1:22379 --advertise-client-urls http://m2.etcd.local:22379 --listen-peer-urls http://127.0.0.1:22380 --initial-advertise-peer-urls=http://m2.etcd.local:22380 --initial-cluster-token tkn --initial-cluster=m1=http://m1.etcd.local:2380,m2=http://m2.etcd.local:22380,m3=http://m3.etcd.local:32380
+
+etcd3: ./etcd --name m3 --data-dir /tmp/m3.data --listen-client-urls http://127.0.0.1:32379 --advertise-client-urls http://m3.etcd.local:32379 --listen-peer-urls http://127.0.0.1:32380 --initial-advertise-peer-urls=http://m3.etcd.local:32380 --initial-cluster-token tkn --initial-cluster=m1=http://m1.etcd.local:2380,m2=http://m2.etcd.local:22380,m3=http://m3.etcd.local:32380

hack/scripts-dev/docker-dns/insecure/run.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+rm -rf /tmp/m1.data /tmp/m2.data /tmp/m3.data
+
+/etc/init.d/bind9 start
+
+# get rid of hosts so go lookup won't resolve 127.0.0.1 to localhost
+cat /dev/null >/etc/hosts
+
+goreman -f /insecure/Procfile start &
+
+# TODO: remove random sleeps
+sleep 7s
+
+ETCDCTL_API=3 ./etcdctl \
+  --endpoints=http://m1.etcd.local:2379 \
+  endpoint health --cluster
+
+ETCDCTL_API=3 ./etcdctl \
+  --endpoints=http://m1.etcd.local:2379,http://m2.etcd.local:22379,http://m3.etcd.local:32379 \
+  put abc def
+
+ETCDCTL_API=3 ./etcdctl \
+  --endpoints=http://m1.etcd.local:2379,http://m2.etcd.local:22379,http://m3.etcd.local:32379 \
+  get abc
+
+# TODO: add host header check to enforce same-origin-policy
+curl -L http://127.0.0.1:2379/v2/keys/queue \
+  -XPOST \
+  -d value=Job1
+
+curl -L http://m1.etcd.local:2379/v2/keys/queue \
+  -XPOST \
+  -d value=Job1

test

@@ -16,10 +16,6 @@ set -e
 
 source ./build
 
-if [[ "${ETCD_SETUP_GOPATH}" == "1" ]]; then
-	etcd_setup_gopath
-fi
-
 # build before setting up test GOPATH
 if [[ "${PASSES}" == *"functional"* ]]; then
 	./tools/functional-tester/build
@@ -281,32 +277,6 @@ function release_pass {
 	mv /tmp/etcd ./bin/etcd-last-release
 }
 
-function gofmt_pass {
-	fmtRes=$(gofmt -l -s -d "${FMT[@]}")
-	if [ -n "${fmtRes}" ]; then
-		echo -e "gofmt checking failed:\n${fmtRes}"
-		exit 255
-	fi
-}
-
-function govet_pass {
-	vetRes=$(go vet "${TEST[@]}")
-	if [ -n "${vetRes}" ]; then
-		echo -e "govet checking failed:\n${vetRes}"
-		exit 255
-	fi
-}
-
-function govet_shadow_pass {
-	fmtpkgs=$(for a in "${FMT[@]}"; do dirname "$a"; done | sort | uniq | grep -v "\\.")
-	fmtpkgs=($fmtpkgs)
-	vetRes=$(go tool vet -all -shadow "${fmtpkgs[@]}" 2>&1 | grep -v '/gw/' || true)
-	if [ -n "${vetRes}" ]; then
-		echo -e "govet -all -shadow checking failed:\n${vetRes}"
-		exit 255
-	fi
-}
-
 function shellcheck_pass {
 	if which shellcheck >/dev/null; then
 		shellcheckResult=$(shellcheck -fgcc build test scripts/* 2>&1 || true)
@@ -364,6 +334,32 @@ function goword_pass {
 	fi
 }
 
+function gofmt_pass {
+	fmtRes=$(gofmt -l -s -d "${FMT[@]}")
+	if [ -n "${fmtRes}" ]; then
+		echo -e "gofmt checking failed:\n${fmtRes}"
+		exit 255
+	fi
+}
+
+function govet_pass {
+	vetRes=$(go vet "${TEST[@]}")
+	if [ -n "${vetRes}" ]; then
+		echo -e "govet checking failed:\n${vetRes}"
+		exit 255
+	fi
+}
+
+function govet_shadow_pass {
+	fmtpkgs=$(for a in "${FMT[@]}"; do dirname "$a"; done | sort | uniq | grep -v "\\.")
+	fmtpkgs=($fmtpkgs)
+	vetRes=$(go tool vet -all -shadow "${fmtpkgs[@]}" 2>&1 | grep -v '/gw/' || true)
+	if [ -n "${vetRes}" ]; then
+		echo -e "govet -all -shadow checking failed:\n${vetRes}"
+		exit 255
+	fi
+}
+
 function gosimple_pass {
 	if which gosimple >/dev/null; then
 		gosimpleResult=$(gosimple "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
@@ -487,13 +483,13 @@ function commit_title_pass {
 function fmt_pass {
 	toggle_failpoints disable
 
-	for p in gofmt \
-			govet \
-			govet_shadow \
-			shellcheck \
+	for p in shellcheck \
 			markdown_you \
 			markdown_marker \
 			goword \
+			gofmt \
+			govet \
+			govet_shadow \
 			gosimple \
 			unused \
 			staticcheck \
@@ -503,9 +499,9 @@ function fmt_pass {
 			receiver_name \
 			commit_title \
 			; do
-		echo "Starting '$p' pass at $(date)"
+		echo "'$p' started at $(date)"
 		"${p}"_pass "$@"
-		echo "Finished '$p' pass at $(date)"
+		echo "'$p' completed at $(date)"
 	done
 }