diff --git a/.gitignore b/.gitignore index 05d2662884..097fd203b0 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ bin/swarmkitstate # ignore code coverage output *coverage.txt + +# dev sync, if used +/.docker-sync/ diff --git a/BUILDING.md b/BUILDING.md index 70e96f5abb..db82902a90 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -113,3 +113,15 @@ NB: As of version 3.0.0-7 the Debian `protobuf-compiler` package lacks a dependency on `libprotobuf-dev` which contains some standard proto definitions, be sure to install both packages. This is [Debian bug #842158](https://bugs.debian.org/842158). + +### Build in a container instead of your local environment + +You can also choose to use a container to build SwarmKit and run tests. Simply +set the `DOCKER_SWARMKIT_USE_CONTAINER` environment variable to any value, +export it, then run `make` targets as you would have done within your local +environment. + +Additionally, if your OS is not Linux, you might want to set and export the +`DOCKER_SWARMKIT_USE_DOCKER_SYNC` environment variable, which will make use of +[docker-sync](https://github.com/EugenMayer/docker-sync) to sync the code to +the container, instead of native mounted volumes. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..4a951ecc69 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +# NOTE(dperny): for some reason, alpine was giving me trouble +FROM golang:1.10.3-stretch + +RUN apt-get update && apt-get install -y make git unzip + +# should stay consistent with the version we use in Circle builds +ARG PROTOC_VERSION=3.5.0 +# make a directory to do these operations in +RUN export PROTOC_TMP_DIR=protoc && mkdir -p $PROTOC_TMP_DIR && cd $PROTOC_TMP_DIR \ + # download the pre-built protoc binary + && curl --silent --show-error --location --output protoc.zip \ + https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/protoc-$PROTOC_VERSION-linux-x86_64.zip \ + # move the binary to /bin. move the well-known types ot /usr/local/include + && unzip protoc.zip && mv bin/protoc /bin/protoc && mv include/* /usr/local/include \ + # remove all of the installation files + && cd .. && rm -rf $PROTOC_TMP_DIR + +WORKDIR /go/src/github.com/docker/swarmkit/ + +# install the dependencies from `make setup` +# we only copy `direct.mk` to avoid busting the cache too easily +COPY direct.mk . +RUN make --file=direct.mk setup + +# now we can copy the rest +COPY . . + +# default to just `make`. If you want to change the default command, change the +# default make command, not this command. +CMD ["make"] diff --git a/Makefile b/Makefile index b9dbdd3d3a..e5e5424f43 100644 --- a/Makefile +++ b/Makefile @@ -24,136 +24,15 @@ VNDR=$(shell which vndr || echo '') GO_LDFLAGS=-ldflags "-X `go list ./version`.Version=$(VERSION)" -.PHONY: clean all AUTHORS fmt vet lint build binaries test integration setup generate protos checkprotos coverage ci check help install uninstall dep-validate -.DEFAULT: default - -all: check binaries test integration ## run fmt, vet, lint, build the binaries and run the tests - -check: fmt vet lint ineffassign misspell ## run fmt, vet, lint, ineffassign, misspell - -ci: check binaries checkprotos coverage coverage-integration ## to be used by the CI - -AUTHORS: .mailmap .git/HEAD - git log --format='%aN <%aE>' | sort -fu > $@ - -# This only needs to be generated by hand when cutting full releases. -version/version.go: - ./version/version.sh > $@ - -setup: ## install dependencies - @echo "🐳 $@" - # TODO(stevvooe): Install these from the vendor directory - @go get -u github.com/golang/lint/golint - #@go get -u github.com/kisielk/errcheck - @go get -u github.com/gordonklaus/ineffassign - @go get -u github.com/client9/misspell/cmd/misspell - @go get -u github.com/lk4d4/vndr - @go get -u github.com/stevvooe/protobuild - -generate: protos - @echo "🐳 $@" - @PATH=${ROOTDIR}/bin:${PATH} go generate -x ${PACKAGES} - -protos: bin/protoc-gen-gogoswarm ## generate protobuf - @echo "🐳 $@" - @PATH=${ROOTDIR}/bin:${PATH} protobuild ${PACKAGES} - -checkprotos: generate ## check if protobufs needs to be generated again - @echo "🐳 $@" - @test -z "$$(git status --short | grep ".pb.go" | tee /dev/stderr)" || \ - ((git diff | cat) && \ - (echo "👹 please run 'make generate' when making changes to proto files" && false)) - -# Depends on binaries because vet will silently fail if it can't load compiled -# imports -vet: binaries ## run go vet - @echo "🐳 $@" - @test -z "$$(go vet ${PACKAGES} 2>&1 | grep -v 'constant [0-9]* not a string in call to Errorf' | egrep -v '(timestamp_test.go|duration_test.go|exit status 1)' | tee /dev/stderr)" - -misspell: - @echo "🐳 $@" - @test -z "$$(find . -type f | grep -v vendor/ | grep -v bin/ | grep -v .git/ | grep -v MAINTAINERS | xargs misspell | tee /dev/stderr)" - -fmt: ## run go fmt - @echo "🐳 $@" - @test -z "$$(gofmt -s -l . | grep -v vendor/ | grep -v ".pb.go$$" | tee /dev/stderr)" || \ - (echo "👹 please format Go code with 'gofmt -s -w'" && false) - @test -z "$$(find . -path ./vendor -prune -o ! -name timestamp.proto ! -name duration.proto -name '*.proto' -type f -exec grep -Hn -e "^ " {} \; | tee /dev/stderr)" || \ - (echo "👹 please indent proto files with tabs only" && false) - @test -z "$$(find . -path ./vendor -prune -o -name '*.proto' -type f -exec grep -Hn "Meta meta = " {} \; | grep -v '(gogoproto.nullable) = false' | tee /dev/stderr)" || \ - (echo "👹 meta fields in proto files must have option (gogoproto.nullable) = false" && false) - -lint: ## run go lint - @echo "🐳 $@" - @test -z "$$(golint ./... | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)" - -ineffassign: ## run ineffassign - @echo "🐳 $@" - @test -z "$$(ineffassign . | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)" - -#errcheck: ## run go errcheck -# @echo "🐳 $@" -# @test -z "$$(errcheck ./... | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)" - -build: ## build the go packages - @echo "🐳 $@" - @go build -i -tags "${DOCKER_BUILDTAGS}" -v ${GO_LDFLAGS} ${GO_GCFLAGS} ${PACKAGES} - -test: ## run tests, except integration tests - @echo "🐳 $@" - @go test -parallel 8 ${RACE} -tags "${DOCKER_BUILDTAGS}" $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES}) - -integration: ## run integration tests - @echo "🐳 $@" - @go test -parallel 8 ${RACE} -tags "${DOCKER_BUILDTAGS}" ${INTEGRATION_PACKAGE} - -FORCE: - -# Build a binary from a cmd. -bin/%: cmd/% FORCE - @test $$(go list) = "${PROJECT_ROOT}" || \ - (echo "👹 Please correctly set up your Go build environment. This project must be located at /src/${PROJECT_ROOT}" && false) - @echo "🐳 $@" - @go build -i -tags "${DOCKER_BUILDTAGS}" -o $@ ${GO_LDFLAGS} ${GO_GCFLAGS} ./$< - -binaries: $(BINARIES) ## build binaries - @echo "🐳 $@" - -clean: ## clean up binaries - @echo "🐳 $@" - @rm -f $(BINARIES) - -install: $(BINARIES) ## install binaries - @echo "🐳 $@" - @mkdir -p $(DESTDIR)/bin - @install $(BINARIES) $(DESTDIR)/bin - -uninstall: - @echo "🐳 $@" - @rm -f $(addprefix $(DESTDIR)/bin/,$(notdir $(BINARIES))) - -coverage: ## generate coverprofiles from the unit tests - @echo "🐳 $@" - @( for pkg in $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES}); do \ - go test -i ${RACE} -tags "${DOCKER_BUILDTAGS}" -test.short -coverprofile="../../../$$pkg/coverage.txt" -covermode=atomic $$pkg || exit; \ - go test ${RACE} -tags "${DOCKER_BUILDTAGS}" -test.short -coverprofile="../../../$$pkg/coverage.txt" -covermode=atomic $$pkg || exit; \ - done ) - -coverage-integration: ## generate coverprofiles from the integration tests - @echo "🐳 $@" - go test ${RACE} -tags "${DOCKER_BUILDTAGS}" -test.short -coverprofile="../../../${INTEGRATION_PACKAGE}/coverage.txt" -covermode=atomic ${INTEGRATION_PACKAGE} - -help: ## this help - @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort - -dep-validate: - @echo "+ $@" - $(if $(VNDR), , \ - $(error Please install vndr: go get github.com/lk4d4/vndr)) - @rm -Rf .vendor.bak - @mv vendor .vendor.bak - @$(VNDR) - @test -z "$$(diff -r vendor .vendor.bak 2>&1 | tee /dev/stderr)" || \ - (echo >&2 "+ inconsistent dependencies! what you have in vendor.conf does not match with what you have in vendor" && false) - @rm -Rf vendor - @mv .vendor.bak vendor +SHELL := /bin/bash + +# stop here. do we want to run everything inside of a container, or do we want +# to run it directly on the host? if the user has set ANY non-empty value for +# the variable DOCKER_SWARMKIT_USE_CONTAINER, then we do all of the making inside of +# a container. We will default to using no container, to avoid breaking +# anyone's workflow +ifdef DOCKER_SWARMKIT_USE_CONTAINER +include containerized.mk +else +include direct.mk +endif diff --git a/containerized.mk b/containerized.mk new file mode 100644 index 0000000000..a2f680d5ec --- /dev/null +++ b/containerized.mk @@ -0,0 +1,47 @@ +IMAGE_NAME=docker/swarmkit +GOPATH=/go +DOCKER_IMAGE_DIR=${GOPATH}/src/${PROJECT_ROOT} + +# don't bother writing every single make target. just pass the call through to +# docker and make +# we prefer `%:` to `.DEFAULT` as the latter doesn't run phony deps +# (see https://www.gnu.org/software/make/manual/html_node/Special-Targets.html) +%:: + @ echo "Running target $@ inside a container" + @ DOCKER_SWARMKIT_DOCKER_RUN_CMD="make $*" $(MAKE) run + +shell: + @ DOCKER_SWARMKIT_DOCKER_RUN_CMD='bash' DOCKER_SWARMKIT_DOCKER_RUN_FLAGS='-i' $(MAKE) run + +.PHONY: image +image: + docker build -t ${IMAGE_NAME} . + +# internal target, only builds the image if it doesn't exist +.PHONY: ensure_image_exists +ensure_image_exists: + @ if [ ! $$(docker images -q ${IMAGE_NAME}) ]; then $(MAKE) image; fi + +# internal target, starts the sync if needed +# uses https://github.com/EugenMayer/docker-sync/blob/47363ee31b71810a60b05822b9c4bd2176951ce8/tasks/sync/sync.thor#L193-L196 +# which is not great, but that's all they expose so far to do this... +.PHONY: ensure_sync_started +ensure_sync_started: + @ kill -0 $$(cat .docker-sync/daemon.pid) 2&> /dev/null || docker-sync start + +# internal target, actually runs a command inside a container +# we don't use the `-i` flag for `docker run` by default as that makes it a pain to kill +# running containers (can't kill with ctrl-c) +.PHONY: run +run: + @ [ "$$DOCKER_SWARMKIT_DOCKER_RUN_CMD" ] || exit 1 + @ $(MAKE) ensure_image_exists + @ DOCKER_RUN_COMMAND="docker run -t -v swarmkit-cache:${GOPATH}" \ + && if [ "$$DOCKER_SWARMKIT_USE_DOCKER_SYNC" ]; then \ + $(MAKE) ensure_sync_started && DOCKER_RUN_COMMAND="$$DOCKER_RUN_COMMAND -v swarmkit-sync:${DOCKER_IMAGE_DIR}"; \ + else \ + DOCKER_RUN_COMMAND="$$DOCKER_RUN_COMMAND -v ${ROOTDIR}:${DOCKER_IMAGE_DIR}"; \ + fi \ + && DOCKER_RUN_COMMAND="$$DOCKER_RUN_COMMAND $$DOCKER_SWARMKIT_DOCKER_RUN_FLAGS ${IMAGE_NAME} $$DOCKER_SWARMKIT_DOCKER_RUN_CMD" \ + && echo $$DOCKER_RUN_COMMAND \ + && $$DOCKER_RUN_COMMAND diff --git a/direct.mk b/direct.mk new file mode 100644 index 0000000000..894e728f4f --- /dev/null +++ b/direct.mk @@ -0,0 +1,156 @@ +.DEFAULT_GOAL = all +.PHONY: all +all: check binaries test integration ## run fmt, vet, lint, build the binaries and run the tests + +.PHONY: check +check: fmt vet lint ineffassign misspell + +.PHONY: ci +ci: check binaries checkprotos coverage coverage-integration ## to be used by the CI + +.PHONY: AUTHORS +AUTHORS: .mailmap .git/HEAD + git log --format='%aN <%aE>' | sort -fu > $@ + +# This only needs to be generated by hand when cutting full releases. +version/version.go: + ./version/version.sh > $@ + +.PHONY: setup +setup: ## install dependencies + @echo "🐳 $@" + # TODO(stevvooe): Install these from the vendor directory + @go get -u github.com/golang/lint/golint + #@go get -u github.com/kisielk/errcheck + @go get -u github.com/gordonklaus/ineffassign + @go get -u github.com/client9/misspell/cmd/misspell + @go get -u github.com/lk4d4/vndr + @go get -u github.com/stevvooe/protobuild + +.PHONY: generate +generate: protos + @echo "🐳 $@" + @PATH=${ROOTDIR}/bin:${PATH} go generate -x ${PACKAGES} + +.PHONY: protos +protos: bin/protoc-gen-gogoswarm ## generate protobuf + @echo "🐳 $@" + @PATH=${ROOTDIR}/bin:${PATH} protobuild ${PACKAGES} + +.PHONY: checkprotos +checkprotos: generate ## check if protobufs needs to be generated again + @echo "🐳 $@" + @test -z "$$(git status --short | grep ".pb.go" | tee /dev/stderr)" || \ + ((git diff | cat) && \ + (echo "👹 please run 'make generate' when making changes to proto files" && false)) + +# Depends on binaries because vet will silently fail if it can't load compiled +# imports +.PHONY: vet +vet: binaries ## run go vet + @echo "🐳 $@" + @test -z "$$(go vet ${PACKAGES} 2>&1 | grep -v 'constant [0-9]* not a string in call to Errorf' | egrep -v '(timestamp_test.go|duration_test.go|exit status 1)' | tee /dev/stderr)" + +.PHONY: misspell +misspell: + @echo "🐳 $@" + @test -z "$$(find . -type f | grep -v vendor/ | grep -v bin/ | grep -v .git/ | grep -v MAINTAINERS | xargs misspell | tee /dev/stderr)" + +.PHONY: fmt +fmt: ## run go fmt + @echo "🐳 $@" + @test -z "$$(gofmt -s -l . | grep -v vendor/ | grep -v ".pb.go$$" | tee /dev/stderr)" || \ + (echo "👹 please format Go code with 'gofmt -s -w'" && false) + @test -z "$$(find . -path ./vendor -prune -o ! -name timestamp.proto ! -name duration.proto -name '*.proto' -type f -exec grep -Hn -e "^ " {} \; | tee /dev/stderr)" || \ + (echo "👹 please indent proto files with tabs only" && false) + @test -z "$$(find . -path ./vendor -prune -o -name '*.proto' -type f -exec grep -Hn "Meta meta = " {} \; | grep -v '(gogoproto.nullable) = false' | tee /dev/stderr)" || \ + (echo "👹 meta fields in proto files must have option (gogoproto.nullable) = false" && false) + +.PHONY: lint +lint: ## run go lint + @echo "🐳 $@" + @test -z "$$(golint ./... | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)" + +.PHONY: ineffassign +ineffassign: ## run ineffassign + @echo "🐳 $@" + @test -z "$$(ineffassign . | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)" + +#errcheck: ## run go errcheck +# @echo "🐳 $@" +# @test -z "$$(errcheck ./... | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)" + +.PHONY: build +build: ## build the go packages + @echo "🐳 $@" + @go build -i -tags "${DOCKER_BUILDTAGS}" -v ${GO_LDFLAGS} ${GO_GCFLAGS} ${PACKAGES} + +.PHONY: test +test: ## run tests, except integration tests + @echo "🐳 $@" + @go test -parallel 8 ${RACE} -tags "${DOCKER_BUILDTAGS}" $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES}) + +.PHONY: integration +integration: ## run integration tests + @echo "🐳 $@" + @go test -parallel 8 ${RACE} -tags "${DOCKER_BUILDTAGS}" ${INTEGRATION_PACKAGE} + +# Build a binary from a cmd. +bin/%: cmd/% .FORCE + @test $$(go list) = "${PROJECT_ROOT}" || \ + (echo "👹 Please correctly set up your Go build environment. This project must be located at /src/${PROJECT_ROOT}" && false) + @echo "🐳 $@" + @go build -i -tags "${DOCKER_BUILDTAGS}" -o $@ ${GO_LDFLAGS} ${GO_GCFLAGS} ./$< + +.PHONY: .FORCE +.FORCE: + +.PHONY: binaries +binaries: $(BINARIES) ## build binaries + @echo "🐳 $@" + +.PHONY: clean +clean: ## clean up binaries + @echo "🐳 $@" + @rm -f $(BINARIES) + +.PHONY: install +install: $(BINARIES) ## install binaries + @echo "🐳 $@" + @mkdir -p $(DESTDIR)/bin + @install $(BINARIES) $(DESTDIR)/bin + +.PHONY: uninstall +uninstall: + @echo "🐳 $@" + @rm -f $(addprefix $(DESTDIR)/bin/,$(notdir $(BINARIES))) + +.PHONY: coverage +coverage: ## generate coverprofiles from the unit tests + @echo "🐳 $@" + @( for pkg in $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES}); do \ + go test -i ${RACE} -tags "${DOCKER_BUILDTAGS}" -test.short -coverprofile="../../../$$pkg/coverage.txt" -covermode=atomic $$pkg || exit; \ + go test ${RACE} -tags "${DOCKER_BUILDTAGS}" -test.short -coverprofile="../../../$$pkg/coverage.txt" -covermode=atomic $$pkg || exit; \ + done ) + +.PHONY: coverage-integration +coverage-integration: ## generate coverprofiles from the integration tests + @echo "🐳 $@" + go test ${RACE} -tags "${DOCKER_BUILDTAGS}" -test.short -coverprofile="../../../${INTEGRATION_PACKAGE}/coverage.txt" -covermode=atomic ${INTEGRATION_PACKAGE} + +.PHONY: help +help: ## this help + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort + +.PHONY: dep-validate +dep-validate: + @echo "+ $@" + $(if $(VNDR), , \ + $(error Please install vndr: go get github.com/lk4d4/vndr)) + @rm -Rf .vendor.bak + @mv vendor .vendor.bak + @$(VNDR) + @test -z "$$(diff -r vendor .vendor.bak 2>&1 | tee /dev/stderr)" || \ + (echo >&2 "+ inconsistent dependencies! what you have in vendor.conf does not match with what you have in vendor" && false) + @rm -Rf vendor + @mv .vendor.bak vendor diff --git a/docker-sync.yml b/docker-sync.yml new file mode 100644 index 0000000000..67856985da --- /dev/null +++ b/docker-sync.yml @@ -0,0 +1,9 @@ +version: "2" + +options: + verbose: true +syncs: + # should stay the same as the volume name used in `containerized.mk`'s `run` target + swarmkit-sync: + src: '.' + sync_excludes: ['_obj', '_test', 'bin']