feat: introduced support for the s3 proxying

.gitignore

@@ -6,3 +6,4 @@ docs/_build
 *.deb
 goiardi-*
 packaging/**/share/goiardi/*
+/.idea

config/config.go

@@ -96,6 +96,7 @@ type Conf struct {
 	StatsdType           string   `toml:"statsd-type"`
 	StatsdInstance       string   `toml:"statsd-instance"`
 	UseS3Upload          bool     `toml:"use-s3-upload"`
+	UseS3Proxy           bool     `toml:"use-s3-proxy"`
 	AWSRegion            string   `toml:"aws-region"`
 	S3Bucket             string   `toml:"s3-bucket"`
 	AWSDisableSSL        bool     `toml:"aws-disable-ssl"`
@@ -212,6 +213,7 @@ type Options struct {
 	StatsdType           string       `long:"statsd-type" description:"statsd format, can be either 'standard' or 'datadog' (default 'standard')" env:"GOIARDI_STATSD_TYPE"`
 	StatsdInstance       string       `long:"statsd-instance" description:"Statsd instance name to use for this server. Defaults to the server's hostname, with '.' replaced by '_'." env:"GOIARDI_STATSD_INSTANCE"`
 	UseS3Upload          bool         `long:"use-s3-upload" description:"Store cookbook files in S3 rather than locally in memory or on disk. This or --local-filestore-dir must be set in SQL mode. Cannot be used with in-memory mode." env:"GOIARDI_USE_S3_UPLOAD"`
+	UseS3Proxy           bool         `long:"use-s3-proxy" description:"When enabled, instead of uploading file directly from client to AWS through the pre-signed url, upload it through goiardi server" env:"GOIARDI_USE_S3_PROXY"`
 	AWSRegion            string       `long:"aws-region" description:"AWS region to use S3 uploads." env:"GOIARDI_AWS_REGION"`
 	S3Bucket             string       `long:"s3-bucket" description:"The name of the S3 bucket storing the files." env:"GOIARDI_S3_BUCKET"`
 	AWSDisableSSL        bool         `long:"aws-disable-ssl" description:"Set to disable SSL for the endpoint. Mostly useful just for testing." env:"GOIARDI_AWS_DISABLE_SSL"`
@@ -517,6 +519,10 @@ func ParseConfigOptions() error {
 			logger.Fatalf("S3 uploads must be used in SQL mode, not in-memory mode.")
 			os.Exit(1)
 		}
+
+		// our legacy behaviour is to always put links for aws s3 upload
+		Config.UseS3Proxy = opts.UseS3Proxy
+
 		if opts.AWSRegion != "" {
 			Config.AWSRegion = opts.AWSRegion
 		}

cookbook/cookbook.go

@@ -23,18 +23,19 @@ package cookbook
 import (
 	"database/sql"
 	"fmt"
+	"net/http"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
 	"github.com/ctdk/goiardi/config"
 	"github.com/ctdk/goiardi/datastore"
 	"github.com/ctdk/goiardi/depgraph"
 	"github.com/ctdk/goiardi/filestore"
 	"github.com/ctdk/goiardi/util"
 	gversion "github.com/hashicorp/go-version"
 	"github.com/tideland/golib/logger"
-	"net/http"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
 )
 
 // cookbook divisions, when resolving cookbook dependencies, that must be filled
@@ -1094,7 +1095,7 @@ func methodize(method string, cbThing []map[string]interface{}) []map[string]int
 			str, _ := j.(string)
 			if k == "url" && (r.MatchString(str) || str == "") {
 				// s3uploads - generate new signed url
-				if config.Config.UseS3Upload {
+				if config.Config.UseS3Upload && !config.Config.UseS3Proxy {
 					var err error
 					retHash[i][k], err = util.S3GetURL("default", chkSum)
 					if err != nil {

datastore/datastore_test.go

@@ -56,7 +56,7 @@ func TestGet(t *testing.T) {
 	}
 	baz := makeDsObj()
 	ds.Set("foo", "bar2", baz)
-	val, found = ds.Get("foo", "bar2")
+	_, found = ds.Get("foo", "bar2")
 	if !found {
 		t.Errorf("Get() did not return a result properly, got '%v' :: %v", val, found)
 	}
@@ -71,7 +71,7 @@ func TestDelete(t *testing.T) {
 		t.Errorf("Couldn't set bar3 baz")
 	}
 	ds.Delete("foo", "bar3")
-	val, found = ds.Get("foo", "bar3")
+	_, found = ds.Get("foo", "bar3")
 	if found {
 		t.Errorf("Delete() did not delete bar3, returned %v!", val)
 	}

file_store.go

@@ -22,9 +22,14 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/tideland/golib/logger"
+
 	"github.com/ctdk/goiardi/config"
 	"github.com/ctdk/goiardi/filestore"
-	"net/http"
+	"github.com/ctdk/goiardi/util"
 )
 
 func fileStoreHandler(w http.ResponseWriter, r *http.Request) {
@@ -37,6 +42,22 @@ func fileStoreHandler(w http.ResponseWriter, r *http.Request) {
 	 * supported. */
 	switch r.Method {
 	case http.MethodGet, http.MethodHead:
+		if config.Config.UseS3Upload {
+			body, err := util.S3FileDownload("default", chksum)
+			if err != nil {
+				w.Header().Set("Content-Type", "application/json")
+				jsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
+				return
+			}
+			w.Header().Set("Content-Type", "application/x-binary")
+			_, err = io.Copy(w, body)
+			if err != nil {
+				logger.Debugf("error while writing response to http handler %+v", err)
+				jsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
+				return
+			}
+			return
+		}
 		w.Header().Set("Content-Type", "application/x-binary")
 		fileStore, err := filestore.Get(chksum)
 		if err != nil {
@@ -47,33 +68,52 @@ func fileStoreHandler(w http.ResponseWriter, r *http.Request) {
 			headResponse(w, r, http.StatusOK)
 			return
 		}
-		w.Write(*fileStore.Data)
+		_, err = w.Write(*fileStore.Data)
+		if err != nil {
+			jsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
+		}
 	case http.MethodPut, http.MethodPost: /* Seems like for file uploads we ought to
 		 * support POST too. */
 		w.Header().Set("Content-Type", "application/json")
+
 		/* Need to distinguish file already existing and some
 		 * sort of error with uploading the file. */
-		if fileStore, _ := filestore.Get(chksum); fileStore != nil {
+		fileExists := false
+		if config.Config.UseS3Upload {
+			fileExists = util.S3FileExists("default", chksum)
+		} else if fileStore, _ := filestore.Get(chksum); fileStore != nil {
+			fileExists = true
+		}
+		if fileExists {
 			fileErr := fmt.Errorf("File with checksum %s already exists.", chksum)
 			/* Send status OK. It seems chef-pedant at least
 			 * tries to upload files twice for some reason.
 			 */
 			jsonErrorReport(w, r, fileErr.Error(), http.StatusOK)
 			return
 		}
+		var err error
 		r.Body = http.MaxBytesReader(w, r.Body, config.Config.ObjMaxSize)
-		fileStore, err := filestore.New(chksum, r.Body, r.ContentLength)
-		if err != nil {
-			jsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
-			return
+		if config.Config.UseS3Upload {
+			//upload file through the s3
+			err = util.S3FileUpload("default", chksum, r.Body)
+		} else {
+			//persist on file system
+			var fileStore *filestore.FileStore
+			fileStore, err = filestore.New(chksum, r.Body, r.ContentLength)
+			if err != nil {
+				jsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
+				return
+			}
+			err = fileStore.Save()
 		}
-		err = fileStore.Save()
 		if err != nil {
 			jsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
 			return
 		}
+
 		fileResponse := make(map[string]string)
-		fileResponse[fileStore.Chksum] = fmt.Sprintf("File with checksum %s uploaded.", fileStore.Chksum)
+		fileResponse[chksum] = fmt.Sprintf("File with checksum %s uploaded.", chksum)
 		enc := json.NewEncoder(w)
 		if err := enc.Encode(&fileResponse); err != nil {
 			jsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)

go.mod

@@ -1,14 +1,16 @@
 module github.com/ctdk/goiardi
 
+go 1.14
+
 require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/DataDog/datadog-go v0.0.0-20160822161430-909c02b65dd8 // indirect
 	github.com/alexcesaro/statsd v2.0.0+incompatible // indirect
-	github.com/aws/aws-sdk-go v1.4.6
+	github.com/aws/aws-sdk-go v1.31.7
 	github.com/ctdk/chefcrypto v0.2.0
 	github.com/ctdk/go-trie v0.0.0-20161110000926-fe74c509b12e
 	github.com/go-ini/ini v1.55.0 // indirect
-	github.com/go-sql-driver/mysql v1.2.1-0.20160802113842-0b58b37b664c
+	github.com/go-sql-driver/mysql v1.5.0
 	github.com/hashicorp/go-version v1.1.0
 	github.com/hashicorp/memberlist v0.2.0 // indirect
 	github.com/hashicorp/serf v0.8.5

go.sum

@@ -11,6 +11,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aws/aws-sdk-go v1.4.6 h1:ZD4LyFcHq1Mz729ZAalUPoPTQMXN/oM3MjpH7/Lqcaw=
 github.com/aws/aws-sdk-go v1.4.6/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
+github.com/aws/aws-sdk-go v1.31.7 h1:TCA+pXKvzDMA3vVqhK21cCy5GarC8pTQb/DrVOWI3iY=
+github.com/aws/aws-sdk-go v1.31.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/ctdk/chefcrypto v0.2.0 h1:Wezo9291QCA2a6As8XtQb4ccPBmng7adthtMUruoMi8=
@@ -27,6 +29,8 @@ github.com/go-ini/ini v1.55.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I
 github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
 github.com/go-sql-driver/mysql v1.2.1-0.20160802113842-0b58b37b664c h1:QD/OSWIQcR3PMs9GzsjN5QOVvxvDI+WrK0GbvNapPds=
 github.com/go-sql-driver/mysql v1.2.1-0.20160802113842-0b58b37b664c/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -120,6 +124,7 @@ github.com/philhofer/fwd v0.0.0-20160129035939-98c11a7a6ec8 h1:jkUFVqrKRttbdDqkT
 github.com/philhofer/fwd v0.0.0-20160129035939-98c11a7a6ec8/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmylund/go-cache v2.0.0+incompatible h1:5ZNJ3XnuQrCjvRXItcjhbD27dVFEk7lVqJHbuI696qc=
@@ -167,6 +172,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwL
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

sandbox/sandbox.go

@@ -205,21 +205,21 @@ func (s *Sandbox) UploadChkList() map[string]map[string]interface{} {
 	chksumStats := make(map[string]map[string]interface{})
 	for _, chk := range s.Checksums {
 		chksumStats[chk] = make(map[string]interface{})
-		var n bool
+		var fileExists bool
 		if config.Config.UseS3Upload {
-			n = util.S3CheckFile("default", chk)
+			fileExists = util.S3FileExists("default", chk)
 		} else {
 			k, _ := filestore.Get(chk)
 			if k != nil {
-				n = true
+				fileExists = true
 			}
 		}
 
-		if n {
+		if fileExists {
 			chksumStats[chk]["needs_upload"] = false
 		} else {
 			// set signed s3 thingamajig here
-			if config.Config.UseS3Upload {
+			if config.Config.UseS3Upload && !config.Config.UseS3Proxy {
 				var err error
 				chksumStats[chk]["url"], err = util.S3PutURL("default", chk)
 				if err != nil {

util/s3.go

@@ -19,19 +19,23 @@ package util
 import (
 	"encoding/base64"
 	"fmt"
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/ctdk/goiardi/config"
-	"github.com/tideland/golib/logger"
+	"io"
 	"net"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/ctdk/goiardi/config"
+	"github.com/tideland/golib/logger"
 )
 
 type s3client struct {
+	awsSession *session.Session
 	bucket     string
 	filePeriod time.Duration
 	s3         *s3.S3
@@ -50,6 +54,7 @@ func InitS3(conf *config.Conf) error {
 	s3cli.bucket = conf.S3Bucket
 	s3cli.filePeriod = time.Duration(conf.S3FilePeriod) * time.Minute
 	s3cli.s3 = s3.New(sess)
+	s3cli.awsSession = sess
 	return nil
 }
 
@@ -64,13 +69,22 @@ func S3GetURL(orgname string, checksum string) (string, error) {
 	return urlStr, err
 }
 
-func S3PutURL(orgname string, checksum string) (string, error) {
+// S3FileDownload downloads file from s3 and returns a readcloser object.
+func S3FileDownload(orgname, checksum string) (io.ReadCloser, error) {
 	key := makeBukkitKey(orgname, checksum)
-	req, _ := s3cli.s3.PutObjectRequest(&s3.PutObjectInput{
+	res, err := s3cli.s3.GetObject(&s3.GetObjectInput{
 		Bucket: aws.String(s3cli.bucket),
 		Key:    aws.String(key),
 	})
+	if err != nil {
+		return nil, err
+	}
+	logger.Debugf("downloading %s file from s3", checksum)
+	return res.Body, nil
+}
 
+// GenerateBase64MD5 converts hex signature provided by chef to a base64 version
+func GenerateBase64MD5(checksum string) (string, error) {
 	// there may be an easier way
 	re := regexp.MustCompile(`[0-9A-Fa-f]{2}`)
 	chopped := re.FindAllString(checksum, -1)
@@ -82,12 +96,25 @@ func S3PutURL(orgname string, checksum string) (string, error) {
 		}
 		b[i] = byte(m)
 	}
-	contentmd5 := base64.StdEncoding.EncodeToString(b)
-	req.HTTPRequest.Header.Set("Content-MD5", contentmd5)
+	return base64.StdEncoding.EncodeToString(b), nil
+}
+func S3PutURL(orgname string, checksum string) (string, error) {
+	key := makeBukkitKey(orgname, checksum)
+	req, _ := s3cli.s3.PutObjectRequest(&s3.PutObjectInput{
+		Bucket: aws.String(s3cli.bucket),
+		Key:    aws.String(key),
+	})
+
+	//amend request
+	contentMD5, err := GenerateBase64MD5(checksum)
+	if err != nil {
+		return "", err
+	}
+	req.HTTPRequest.Header.Set("Content-MD5", contentMD5)
 	req.HTTPRequest.URL.Host = s3cli.makeHostPort(req.HTTPRequest.URL.Host)
 
 	urlStr, err := req.Presign(s3cli.filePeriod)
-	logger.Debugf("presign: %s %s", urlStr, contentmd5)
+	logger.Debugf("presign: %s %s %s", urlStr, contentMD5, checksum)
 	return urlStr, err
 }
 
@@ -134,17 +161,35 @@ func S3DeleteHashes(fileHashes []string) {
 	}
 }
 
-func S3CheckFile(orgname, checksum string) bool {
+// S3FileUpload uploads file to s3 fileSystem.
+func S3FileUpload(orgName string, chksum string, body io.ReadCloser) error {
+	md5, err := GenerateBase64MD5(chksum)
+	if err != nil {
+		return err
+	}
+	uploader := s3manager.NewUploader(s3cli.awsSession)
+	upParams := &s3manager.UploadInput{
+		Bucket:     aws.String(s3cli.bucket),
+		Key:        aws.String(makeBukkitKey(orgName, chksum)),
+		Body:       body,
+		ContentMD5: aws.String(md5),
+	}
+	result, err := uploader.Upload(upParams)
+	if err != nil {
+		return err
+	}
+	logger.Debugf("s3 upload result for file %s %+v", chksum, result)
+	return nil
+}
+
+// S3FileExists checks if such file exists on s3, returns true if it is.
+func S3FileExists(orgname, checksum string) bool {
 	params := &s3.HeadObjectInput{
 		Bucket: aws.String(s3cli.bucket),
 		Key:    aws.String(makeBukkitKey(orgname, checksum)),
 	}
 	_, err := s3cli.s3.HeadObject(params)
-	var ret bool
-	if err == nil {
-		ret = true
-	}
-	return ret
+	return err == nil
 }
 
 func makeBukkitKey(orgname, checksum string) string {