feat: Better lint and test for devs and Makefile cleanup. (project-st…

…acker#511)

This change does several things all towards the end of making
it easier for developers to run tests and lint on go code.

Now a developer can run either 'make go-test' or 'make lint'
without having lots of C dependencies or running in a specific
container or stacker.

Things changed:

 * Add a build tag 'skipembed' that builds without needing
   cmd/stacker/lxc-wrapper/lxc-wrapper . This allows you to
   build (or test) without that file.  In order to ensure that
   you do not get very far trying to run a 'stacker' binary,
   I've added a 'panic' if stacker binary is built with skipembed.

 * Add download-tools target to makefile to ease in downloading of
   regctl and zot.

 * Move the downloading of golangci-lint from the github workflow
   to Makefile.  This makes it easier for developer to get it.
   It is also added to the 'download-tools' target.

 * be specific about the version of golangci-lint that is used.
   1.54.2.  golangci-lint's docs specifically say to do this:

    > IMPORTANT: It's highly recommended installing a specific version
    > of golangci-lint available on the releases page.

   changing it is simply a matter of changing the value in the makefile.

 * Move running of 'go test' out of the lint target to its own
   'go-test' target.  go-test target will now write a hack/coverage.html
   so you can view coverage easily.

 * add 'dlbin' makefile "function" for downloading binaries, and use
   that for regctl and zot from their rules.

 * rename TOOLS_DIR to TOOLS_D (consistency with BUILD_D, and just
   shorter)

Signed-off-by: Scott Moser <smoser@brickies.net>

.github/workflows/build.yaml

@@ -65,9 +65,9 @@ jobs:
           sudo add-apt-repository -y ppa:project-machine/squashfuse
           sudo apt-get update
           sudo apt-get install -yy lxc-utils lxc-dev libacl1-dev jq libcap-dev libseccomp-dev libpam-dev bats parallel libzstd-dev
-          GO111MODULE=off go get github.com/opencontainers/umoci/cmd/umoci
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin
           sudo apt-get install -yy autoconf automake make autogen autoconf libtool binutils git squashfs-tools libcryptsetup-dev libdevmapper-dev cryptsetup-bin squashfuse
+          GO111MODULE=off go get github.com/opencontainers/umoci/cmd/umoci
+          make download-tools
           echo "running kernel is: $(uname -a)"
       - name: Go-download
         run: |

.gitignore

@@ -3,6 +3,8 @@
 /stacker
 /stacker-dynamic
 .build
+coverage.txt
+hack/
 
 # IDEs
 .vscode

Makefile

@@ -24,14 +24,18 @@ STACKER_BUILD_UBUNTU_IMAGE?=$(STACKER_DOCKER_BASE)ubuntu:latest
 LXC_CLONE_URL?=https://github.com/lxc/lxc
 LXC_BRANCH?=stable-5.0
 
+HACK_D := $(TOP_LEVEL)/hack
 # helper tools
-TOOLSDIR := $(shell pwd)/hack/tools
-REGCLIENT := $(TOOLSDIR)/bin/regctl
+TOOLS_D := $(HACK_D)/tools
+REGCLIENT := $(TOOLS_D)/bin/regctl
 REGCLIENT_VERSION := v0.5.1
 # OCI registry
-ZOT := $(TOOLSDIR)/bin/zot
+ZOT := $(TOOLS_D)/bin/zot
 ZOT_VERSION := 2.0.0-rc6
 
+GOLANGCI_LINT_VERSION = v1.54.2
+GOLANGCI_LINT = $(TOOLS_D)/golangci-lint/$(GOLANGCI_LINT_VERSION)/golangci-lint
+
 STAGE1_STACKER ?= ./stacker-dynamic
 
 STACKER_DEPS = $(GO_SRC) go.mod go.sum
@@ -61,30 +65,44 @@ go-download:
 	go mod download
 
 .PHONY: lint
-lint: cmd/stacker/lxc-wrapper/lxc-wrapper $(GO_SRC)
+lint: $(GO_SRC) $(GOLANGCI_LINT)
 	go mod tidy
 	go fmt ./... && ([ -z $(CI) ] || git diff --exit-code)
 	bash test/static-analysis.sh
-	go test -v -trimpath -cover -coverpkg stackerbuild.io/stacker/./... -coverprofile=coverage.txt -covermode=atomic -tags "$(BUILD_TAGS)" stackerbuild.io/stacker/./...
-	$(shell go env GOPATH)/bin/golangci-lint run --build-tags "$(BUILD_TAGS)"
+	$(GOLANGCI_LINT) run --build-tags "$(BUILD_TAGS) skipembed"
+
+.PHONY: go-test
+go-test:
+	go test -v -trimpath -cover -coverprofile=coverage.txt -covermode=atomic -tags "exclude_graphdriver_btrfs exclude_graphdriver_devicemapper containers_image_openpgp osusergo netgo skipembed" ./pkg/... ./cmd/...
+	go tool cover -html coverage.txt  -o $(HACK_D)/coverage.html
+
+.PHONY: download-tools
+download-tools: $(GOLANGCI_LINT) $(REGCLIENT) $(ZOT)
+
+$(GOLANGCI_LINT):
+	@mkdir -p $(dir $@)
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(dir $@)"
+	@mkdir -p "$(TOOLS_D)/bin"
+	ln -sf "$@" "$(TOOLS_D)/bin/"
+
+# dlbin is used with $(call dlbin,path,url)
+# it downloads a url to path and makes it executable.
+# it creates dest dir and atomically moves into place. t gets <name>.pid
+dlbin = set -x; mkdir -p $(dir $1) && t=$1.$$$$ && curl -Lo "$$t" "$2" && chmod +x "$$t" && mv "$$t" "$1"
 
 $(REGCLIENT):
-	mkdir -p $(TOOLSDIR)/bin
-	curl -Lo $(REGCLIENT) https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64
-	chmod +x $(REGCLIENT)
+	$(call dlbin,$@,https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64)
 
 $(ZOT):
-	mkdir -p $(TOOLSDIR)/bin
-	curl -Lo $(ZOT) https://github.com/project-zot/zot/releases/download/v$(ZOT_VERSION)/zot-linux-amd64-minimal
-	chmod +x $(ZOT)
+	$(call dlbin,$@,https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64)
 
 TEST?=$(patsubst test/%.bats,%,$(wildcard test/*.bats))
 PRIVILEGE_LEVEL?=
 
 # make check TEST=basic will run only the basic test
 # make check PRIVILEGE_LEVEL=unpriv will run only unprivileged tests
 .PHONY: check
-check: lint test
+check: lint test go-test
 
 .PHONY: test
 test: stacker $(REGCLIENT) $(ZOT)

cmd/stacker/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"embed"
 	"fmt"
 	"os"
 	"os/exec"
@@ -28,9 +27,6 @@ var (
 	lxc_version = ""
 )
 
-//go:embed lxc-wrapper/lxc-wrapper
-var embeddedFS embed.FS
-
 func shouldShowProgress(ctx *cli.Context) bool {
 	/* if the user provided explicit recommendations, follow those */
 	if ctx.Bool("no-progress") {
@@ -84,6 +80,9 @@ func shouldSkipInternalUserns(ctx *cli.Context) bool {
 }
 
 func main() {
+	if !hasEmbedded {
+		panic("stacker was built without embedded binaries.")
+	}
 	sigquits := make(chan os.Signal, 1)
 	go func() {
 		for range sigquits {

cmd/stacker/main_embed.go

@@ -0,0 +1,10 @@
+//go:build !skipembed
+
+package main
+
+import "embed"
+
+//go:embed lxc-wrapper/lxc-wrapper
+var embeddedFS embed.FS
+
+const hasEmbedded = true

cmd/stacker/main_noembed.go

@@ -0,0 +1,9 @@
+//go:build skipembed
+
+package main
+
+import "embed"
+
+var embeddedFS embed.FS
+
+const hasEmbedded = true