Merge pull request #1 from imeoer/main

Converter: implement conversion framework

.gitignore

@@ -14,4 +14,5 @@
 # Dependency directories (remove the comment below to include it)
 # vendor/
 
+/misc/config/config.yaml
 /acceleration-service

.golangci.yml

@@ -0,0 +1,22 @@
+# https://golangci-lint.run/usage/configuration#config-file
+
+linters:
+  enable:
+    - structcheck
+    - varcheck
+    - staticcheck
+    - unconvert
+    - gofmt
+    - goimports
+    - revive
+    - ineffassign
+    - vet
+    - unused
+    - misspell
+  disable:
+    - errcheck
+
+run:
+  deadline: 4m
+  skip-dirs:
+    - misc

Makefile

@@ -2,5 +2,27 @@ GIT_COMMIT := $(shell git rev-list -1 HEAD)
 BUILD_TIME := $(shell date -u +%Y%m%d.%H%M)
 CWD := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 
-default:
+# Build binary to ./acceleration-service
+default: check
 	go build -ldflags '-X main.versionGitCommit=${GIT_COMMIT} -X main.versionBuildTime=${BUILD_TIME}' -gcflags=all="-N -l" ./cmd/acceleration-service
+
+install-check-tools:
+	@curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.42.1
+
+check:
+	@echo "$@"
+	@$(shell go env GOPATH)/bin/golangci-lint run
+	@$(shell go env GOPATH)/bin/golangci-lint run
+
+# Run unit testing
+# Run a particular test in this way:
+# go test -v -count=1 -run TestFoo ./pkg/...
+ut: default
+	go test -count=1 -v ./pkg/...
+
+# Run integration testing
+smoke: default
+	go test -count=1 -v ./test
+
+# Run testing 
+test: default ut smoke

README.md

@@ -1,2 +1,93 @@
-# acceleration-service
-Provides a general service to support image acceleration based on kinds of accelerator like Nydus and eStargz etc.
+# Acceleration Service
+
+Acceleration Service provides a general service to Harbor with the ability to automatically convert user images to accelerated images. When a user does something such as artifact push, Harbor will request the service to complete the corresponding image conversion through its integrated Nydus, eStargz, etc. drivers.
+
+⚠️ This project is still in progress, see more details in this [proposal](https://github.com/goharbor/community/pull/167).
+
+## Quickstart
+
+### Build binary
+
+``` shell
+$ make
+```
+
+### Configuration (Harbor side)
+
+Login to Harbor web interface to complete the following operation.
+
+1. Add webhook configuration
+
+Projects -> Webhooks -> `+ NEW WEBHOOK`, add webhook configuration with following fields:
+
+- Notify Type: http
+- Event Type: artifact pushed
+- Endpoint URL: <acceleration service address>/api/v1/conversions
+- Auth Header: <configured in acceleration service>
+
+2. Add robot account
+
+Administration -> Robot Accounts -> `+ NEW ROBOT ACCOUNT`, add robot account with following fields:
+
+- Expiration time: <by your chose>
+- Reset permissions: select `Push Artifact`, `Pull Artifact`, `Create Tag`
+
+Generate a base64 encoded auth string using robot account name and secret like this:
+
+``` shell
+$ printf '<robot-name>:<robot-secret>' | base64
+```
+
+### Configuration (Acceleration service side)
+
+1. Copy template config file
+
+``` shell
+$ cp misc/config/config.yaml.tmpl misc/config/config.yaml
+```
+
+2. Edit config file
+
+Following comments in `misc/config/config.yaml`, ensure base64 encoded auth string and webhook auth header be configured correctly.
+
+### Boot service
+
+1. Ensure containerd service is running
+
+Acceleration service depends on containerd service to manage image content, containerd service is used default if you have installed the latest docker.
+
+``` shell
+$ ps aux | grep contained
+root        929  0.5  0.5 2222284 22368 ?       Ssl  Sep16   4:27 /usr/local/bin/containerd
+```
+
+2. Boot daemon to serve webhook request in PUSH_ARTIFACT event
+
+``` shell
+$ ./acceleration-service --config ./misc/config/config.yaml
+```
+
+### Test with pushing image
+
+1. Push a test image to Harbor
+
+``` shell
+$ docker push <harbor-service-address>/library/nginx:latest
+```
+
+2. Got converted image
+
+If everything is ready, we can see the log output in acceleration service like this:
+
+``` shell
+INFO[2021-09-17T05:43:49.783769943Z] Version: ecbfefc312ea78d81eb895ef7768a60caa30a281.20210917.0543
+INFO[2021-09-17T05:43:49.786146553Z] [API] HTTP server started on 0.0.0.0:2077
+INFO[2021-09-17T05:44:01.82592358Z] received webhook request from 192.168.1.1:46664  module=api
+INFO[2021-09-17T05:44:01.826490232Z] POST /api/v1/conversions 200 552.695µs 392>5bytes 192.168.1.1  module=api
+INFO[2021-09-17T05:44:01.830893103Z] pulling image 192.168.1.1/library/nginx:latest  module=converter
+INFO[2021-09-17T05:44:01.916123236Z] pulled image 192.168.1.1/library/nginx:latest  module=converter
+INFO[2021-09-17T05:44:01.916175337Z] converting image 192.168.1.1/library/nginx:latest  module=converter
+INFO[2021-09-17T05:44:04.317992888Z] converted image 192.168.1.1/library/nginx:latest-converted  module=converter
+```
+
+After that, we will find converted artifact in Harbor interface with name `<harbor-service-address>/nginx:latest-converted`.

cmd/acceleration-service/main.go

@@ -17,6 +17,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -31,7 +32,8 @@ var versionBuildTime string
 
 func main() {
 	logrus.SetFormatter(&logrus.TextFormatter{
-		FullTimestamp: true,
+		FullTimestamp:   true,
+		TimestampFormat: time.RFC3339Nano,
 	})
 
 	version := fmt.Sprintf("%s.%s", versionGitCommit, versionBuildTime)
@@ -52,11 +54,11 @@ func main() {
 
 			d, err := daemon.NewDaemon(cfg)
 			if err != nil {
-				return errors.Wrapf(err, "create daemon")
+				return errors.Wrap(err, "create daemon")
 			}
 
 			if err := d.Run(); err != nil {
-				return errors.Wrapf(err, "run daemon")
+				return errors.Wrap(err, "run daemon")
 			}
 
 			return nil

go.mod

@@ -3,11 +3,19 @@ module github.com/goharbor/acceleration-service
 go 1.16
 
 require (
+	github.com/containerd/containerd v1.5.7
+	github.com/docker/cli v20.10.9+incompatible
+	github.com/goharbor/harbor/src v0.0.0-20211021012518-bc6a7f65a6fa
 	github.com/labstack/echo-contrib v0.11.0
-	github.com/labstack/echo/v4 v4.5.0
+	github.com/labstack/echo/v4 v4.6.1
+	github.com/labstack/gommon v0.3.0
+	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/image-spec v1.0.1
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.10.0
+	github.com/prometheus/client_golang v1.11.0
 	github.com/sirupsen/logrus v1.8.1
+	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli/v2 v2.3.0
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )