From feff92075c3c2b29e3f68829917ef6d102555e30 Mon Sep 17 00:00:00 2001 From: Baha Shaaban Date: Tue, 8 Nov 2022 11:22:57 -0500 Subject: [PATCH] feat: initial cmdutil-go code Signed-off-by: Baha Shaaban --- .codecov.yaml | 13 ++ .gitignore | 33 +++ .golangci.yml | 240 +++++++++++++++++++++ Makefile | 38 ++++ README.md | 11 + go.mod | 20 ++ go.sum | 37 ++++ pkg/tls/certpool.go | 164 ++++++++++++++ pkg/tls/certpool_test.go | 427 +++++++++++++++++++++++++++++++++++++ pkg/utils/cmd/util.go | 152 +++++++++++++ pkg/utils/cmd/util_test.go | 266 +++++++++++++++++++++++ pkg/utils/tls/util.go | 45 ++++ pkg/utils/tls/util_test.go | 69 ++++++ scripts/check_license.sh | 66 ++++++ scripts/check_lint.sh | 19 ++ scripts/check_unit.sh | 29 +++ 16 files changed, 1629 insertions(+) create mode 100644 .codecov.yaml create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 Makefile create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pkg/tls/certpool.go create mode 100644 pkg/tls/certpool_test.go create mode 100644 pkg/utils/cmd/util.go create mode 100644 pkg/utils/cmd/util_test.go create mode 100644 pkg/utils/tls/util.go create mode 100644 pkg/utils/tls/util_test.go create mode 100755 scripts/check_license.sh create mode 100755 scripts/check_lint.sh create mode 100755 scripts/check_unit.sh diff --git a/.codecov.yaml b/.codecov.yaml new file mode 100644 index 0000000..d533ec7 --- /dev/null +++ b/.codecov.yaml @@ -0,0 +1,13 @@ +# Copyright SecureKey Technologies Inc. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +coverage: + status: + project: + default: + target: 85% + patch: + default: + target: 85% + only_pulls: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0fb2c5c --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# +# Copyright SecureKey Technologies Inc. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Editor and go temporary files & folders +.swp +vendor + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out +.idea +.DS_Store + +coverage.out +coverage.txt + +# Exclude build directory +.build +*.log +build/ + diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..85b88b3 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,240 @@ +# +# Copyright SecureKey Technologies Inc. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# + +run: + # Timeout for analysis, e.g. 30s, 5m. + # Default: 1m + timeout: 5m + + # Define the Go version limit. + # Mainly related to generics support in go1.18. + # Default: use Go version from the go.mod file, fallback on the env var `GOVERSION`, fallback on 1.19 + go: "1.19" + + +# All possible options can be found here: https://github.com/golangci/golangci-lint/blob/master/.golangci.reference.yml +linters-settings: + cyclop: + # The maximal code complexity to report. + # Default: 10 + max-complexity: 30 + # The maximal average package complexity. + # If it's higher than 0.0 (float) the check is enabled + # Default: 0.0 + package-average: 10.0 + + errcheck: + # Report about not checking of errors in type assertions: `a := b.(MyStruct)`. + # Such cases aren't reported by default. + # Default: false + check-type-assertions: true + + funlen: + # Checks the number of lines in a function. + # If lower than 0, disable the check. + # Default: 60 + lines: 100 + # Checks the number of statements in a function. + # If lower than 0, disable the check. + # Default: 40 + statements: 50 + + gocognit: + # Minimal code complexity to report + # Default: 30 (but we recommend 10-20) + min-complexity: 20 + + gocritic: + # Settings passed to gocritic. + # The settings key is the name of a supported gocritic checker. + # The list of supported checkers can be find in https://go-critic.github.io/overview. + settings: + captLocal: + # Whether to restrict checker to params only. + # Default: true + paramsOnly: false + underef: + # Whether to skip (*x).method() calls where x is a pointer receiver. + # Default: true + skipRecvDeref: false + + gomnd: + # List of function patterns to exclude from analysis. + # Values always ignored: `time.Date` + # Default: [] + ignored-functions: + - os.Chmod + - os.Mkdir + - os.MkdirAll + - os.OpenFile + - os.WriteFile + - prometheus.ExponentialBuckets + - prometheus.ExponentialBucketsRange + - prometheus.LinearBuckets + - strconv.FormatFloat + - strconv.FormatInt + - strconv.FormatUint + - strconv.ParseFloat + - strconv.ParseInt + - strconv.ParseUint + + gomodguard: + blocked: + # List of blocked modules. + # Default: [] + modules: + - github.com/golang/protobuf: + recommendations: + - google.golang.org/protobuf + reason: "see https://developers.google.com/protocol-buffers/docs/reference/go/faq#modules" + - github.com/satori/go.uuid: + recommendations: + - github.com/google/uuid + reason: "satori's package is not maintained" + - github.com/gofrs/uuid: + recommendations: + - github.com/google/uuid + reason: "see recommendation from dev-infra team: https://confluence.gtforge.com/x/gQI6Aw" + + govet: + # Enable all analyzers. + # Default: false + enable-all: true + # Disable analyzers by name. + # Run `go tool vet help` to see all analyzers. + # Default: [] + disable: + - fieldalignment # too strict + # Settings per analyzer. + settings: + shadow: + # Whether to be strict about shadowing; can be noisy. + # Default: false + strict: false + + nakedret: + # Make an issue if func has more lines of code than this setting, and it has naked returns. + # Default: 30 + max-func-lines: 0 + + nolintlint: + # Exclude following linters from requiring an explanation. + # Default: [] + allow-no-explanation: [ funlen, gocognit, lll ] + # Enable to require an explanation of nonzero length after each nolint directive. + # Default: false + require-explanation: true + # Enable to require nolint directives to mention the specific linter being suppressed. + # Default: false + require-specific: true + + rowserrcheck: + # database/sql is always checked + # Default: [] + packages: + - github.com/jmoiron/sqlx + + tenv: + # The option `all` will run against whole test files (`_test.go`) regardless of method/function signatures. + # Otherwise, only methods that take `*testing.T`, `*testing.B`, and `testing.TB` as arguments are checked. + # Default: false + all: true + + varcheck: + # Check usage of exported fields and variables. + # Default: false + exported-fields: false # default false # TODO: enable after fixing false positives + + +linters: + disable-all: true + enable: + ## enabled by default + - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases + - gosimple # Linter for Go source code that specializes in simplifying a code + - govet # Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string + - ineffassign # Detects when assignments to existing variables are not used + - staticcheck # Staticcheck is a go vet on steroids, applying a ton of static analysis checks + - typecheck # Like the front-end of a Go compiler, parses and type-checks Go code + - unused # Checks Go code for unused constants, variables, functions and types + ## disabled by default + - asasalint # Check for pass []any as any in variadic func(...any) + - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers + - bidichk # Checks for dangerous unicode character sequences + - bodyclose # checks whether HTTP response body is closed successfully + - contextcheck # check the function whether use a non-inherited context + - cyclop # checks function and package cyclomatic complexity + - dupl # Tool for code clone detection + - durationcheck # check for two durations multiplied together + - errname # Checks that sentinel errors are prefixed with the Err and error types are suffixed with the Error. + - errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13. + - execinquery # execinquery is a linter about query string checker in Query function which reads your Go src files and warning it finds + - exhaustive # check exhaustiveness of enum switch statements + - exportloopref # checks for pointers to enclosing loop variables + - forbidigo # Forbids identifiers + - funlen # Tool for detection of long functions + - gochecknoglobals # check that no global variables exist + - gochecknoinits # Checks that no init functions are present in Go code + - gocognit # Computes and checks the cognitive complexity of functions + - goconst # Finds repeated strings that could be replaced by a constant + - gocritic # Provides diagnostics that check for bugs, performance and style issues. + - gocyclo # Computes and checks the cyclomatic complexity of functions + - godot # Check if comments end in a period + - goimports # In addition to fixing imports, goimports also formats your code in the same style as gofmt. + - gomnd # An analyzer to detect magic numbers. + - gomodguard # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations. + - goprintffuncname # Checks that printf-like functions are named with f at the end + - gosec # Inspects source code for security problems + - lll # Reports long lines + - makezero # Finds slice declarations with non-zero initial length + - nakedret # Finds naked returns in functions greater than a specified function length + - nestif # Reports deeply nested if statements + - nilerr # Finds the code that returns nil even if it checks that the error is not nil. + - nilnil # Checks that there is no simultaneous return of nil error and an invalid value. + - noctx # noctx finds sending http request without context.Context +# - nolintlint # Reports ill-formed or insufficient nolint directives + - nonamedreturns # Reports all named returns + - nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL. + - predeclared # find code that shadows one of Go's predeclared identifiers + - promlinter # Check Prometheus metrics naming via promlint + - revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint. + - rowserrcheck # checks whether Err of rows is checked successfully + - sqlclosecheck # Checks that sql.Rows and sql.Stmt are closed. + - stylecheck # Stylecheck is a replacement for golint + - tenv # tenv is analyzer that detects using os.Setenv instead of t.Setenv since Go1.17 +# - testpackage # linter that makes you use a separate _test package + - tparallel # tparallel detects inappropriate usage of t.Parallel() method in your Go test codes + - unconvert # Remove unnecessary type conversions + - unparam # Reports unused function parameters + - wastedassign # wastedassign finds wasted assignment statements. + - whitespace # Tool for detection of leading and trailing whitespace + +issues: + # Maximum count of issues with the same text. + # Set to 0 to disable. + # Default: 3 + max-same-issues: 0 + + exclude-rules: + - source: "^//\\s*go:generate\\s" + linters: [ lll ] + - source: "(noinspection|TODO)" + linters: [ godot ] + - source: "//noinspection" + linters: [ gocritic ] + - source: "^\\s+if _, ok := err\\.\\([^.]+\\.InternalError\\); ok {" + linters: [ errorlint ] + - text: logger is a global variable + linters: [ gochecknoglobals ] + - path: "_test\\.go" + linters: + - bodyclose + - dupl + - funlen + - goconst + - gosec + - noctx + - wrapcheck diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..581460f --- /dev/null +++ b/Makefile @@ -0,0 +1,38 @@ +# Copyright SecureKey Technologies Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +GOBIN_PATH=$(abspath .)/.build/bin + +# Tool commands (overridable) +GO_VER ?= 1.19 + +OS := $(shell uname) +ifeq ($(OS),$(filter $(OS),Darwin Linux)) + PATH:=$(PATH):$(GOBIN_PATH) +else + PATH:=$(PATH);$(subst /,\\,$(GOBIN_PATH)) +endif + +.PHONY: all +all: clean checks unit-test + +.PHONY: checks +checks: license lint + +.PHONY: lint +lint: + @scripts/check_lint.sh + +.PHONY: license +license: + @scripts/check_license.sh + +.PHONY: unit-test +unit-test: + @scripts/check_unit.sh + +.PHONY: clean +clean: + @rm -rf ./build + @rm -rf coverage*.out diff --git a/README.md b/README.md index 7cc14e5..fb1a035 100644 --- a/README.md +++ b/README.md @@ -1 +1,12 @@ +[![Release](https://img.shields.io/github/release/trustbloc/cmdutil-go.svg?style=flat-square)](https://github.com/trustbloc/cmdutil-go/releases/latest) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://raw.githubusercontent.com/trustbloc/cmdutil-go/main/LICENSE) +[![Godocs](https://img.shields.io/badge/godoc-reference-blue.svg)](https://godoc.org/github.com/trustbloc/cmdutil-go) + +[![Build Status](https://github.com/trustbloc/cmdutil-go/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/trustbloc/cmdutil-go/actions/workflows/build.yml) +[![codecov](https://codecov.io/gh/trustbloc/cmdutil-go/branch/main/graph/badge.svg)](https://codecov.io/gh/trustbloc/cmdutil-go) +[![Go Report Card](https://goreportcard.com/badge/github.com/trustbloc/cmdutil-go)](https://goreportcard.com/report/github.com/trustbloc/cmdutil-go) + # cmdutil-go + +General purpose controller commands utility module. + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9275963 --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module github.com/trustbloc/cmdutil-go + +go 1.19 + +require ( + github.com/spf13/cobra v1.6.1 + github.com/stretchr/testify v1.8.1 + github.com/trustbloc/logutil-go v0.0.0-20221107142326-c9110e31ee60 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.23.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c73cfe9 --- /dev/null +++ b/go.sum @@ -0,0 +1,37 @@ +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/trustbloc/logutil-go v0.0.0-20221107142326-c9110e31ee60 h1:Y0w1bvjDTjze0YR2jQxiC+f2benhAMN04w1iwIuDiJo= +github.com/trustbloc/logutil-go v0.0.0-20221107142326-c9110e31ee60/go.mod h1:HRaXVV1caceumbDBwLO3ByiCcAc18KwrNvZ7JQBvDIQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= +go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/tls/certpool.go b/pkg/tls/certpool.go new file mode 100644 index 0000000..545fcc0 --- /dev/null +++ b/pkg/tls/certpool.go @@ -0,0 +1,164 @@ +/* +Copyright SecureKey Technologies Inc. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package tls + +import ( + "crypto/x509" + "sync" + "sync/atomic" + + "github.com/trustbloc/logutil-go/pkg/log" +) + +var logger = log.New("cmdutil-go-tls") + +// CertPool is a thread safe wrapper around the x509 standard library +// cert pool implementation. +// It optionally allows loading the system trust store. +type CertPool struct { + certPool *x509.CertPool + certs []*x509.Certificate + certsByName map[string][]int + lock sync.RWMutex + dirty int32 + systemCertPool bool +} + +// NewCertPool new CertPool implementation. +func NewCertPool(useSystemCertPool bool) (*CertPool, error) { + c, err := loadSystemCertPool(useSystemCertPool) + if err != nil { + return nil, err + } + + newCertPool := &CertPool{ + certsByName: make(map[string][]int), + certPool: c, + systemCertPool: useSystemCertPool, + } + + return newCertPool, nil +} + +// Get returns certpool. +// If there are any certs in cert queue added by any previous Add() call +// it adds those certs to certpool before returning. +func (c *CertPool) Get() (*x509.CertPool, error) { + // if dirty then add certs from queue to cert pool + if atomic.CompareAndSwapInt32(&c.dirty, 1, 0) { + // swap certpool if queue is dirty + err := c.swapCertPool() + if err != nil { + return nil, err + } + } + + c.lock.RLock() + defer c.lock.RUnlock() + + return c.certPool, nil +} + +// Add adds given certs to cert pool queue, those certs will be added to certpool during subsequent Get() call. +func (c *CertPool) Add(certs ...*x509.Certificate) { + if len(certs) == 0 { + return + } + + // filter certs to be added, check if they already exist or duplicate + certsToBeAdded := c.filterCerts(certs...) + + if len(certsToBeAdded) > 0 { + c.lock.Lock() + defer c.lock.Unlock() + + for _, newCert := range certsToBeAdded { + // Store cert name index + name := string(newCert.RawSubject) + c.certsByName[name] = append(c.certsByName[name], len(c.certs)) + // Store cert + c.certs = append(c.certs, newCert) + } + + atomic.CompareAndSwapInt32(&c.dirty, 0, 1) + } +} + +func (c *CertPool) swapCertPool() error { + newCertPool, err := loadSystemCertPool(c.systemCertPool) + if err != nil { + return err + } + + c.lock.Lock() + defer c.lock.Unlock() + + // add all new certs in queue to new cert pool + for _, cert := range c.certs { + newCertPool.AddCert(cert) + } + + // swap old certpool with new one + c.certPool = newCertPool + + return nil +} + +// filterCerts remove certs from list if they already exist in pool or duplicate. +func (c *CertPool) filterCerts(certs ...*x509.Certificate) []*x509.Certificate { + c.lock.RLock() + defer c.lock.RUnlock() + + filtered := []*x509.Certificate{} + +CertLoop: + for _, cert := range certs { + if cert == nil { + continue + } + possibilities := c.certsByName[string(cert.RawSubject)] + for _, p := range possibilities { + if c.certs[p].Equal(cert) { + continue CertLoop + } + } + filtered = append(filtered, cert) + } + + // remove duplicate from list of certs being passed + return removeDuplicates(filtered...) +} + +func removeDuplicates(certs ...*x509.Certificate) []*x509.Certificate { + encountered := map[*x509.Certificate]bool{} + result := []*x509.Certificate{} + + for v := range certs { + if !encountered[certs[v]] { + encountered[certs[v]] = true + + result = append(result, certs[v]) + } + } + + return result +} + +func loadSystemCertPool(useSystemCertPool bool) (*x509.CertPool, error) { + if !useSystemCertPool { + return x509.NewCertPool(), nil + } + + systemCertPool, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + + //nolint:staticcheck + logger.Debug("Loaded system cert pool of size", log.WithCertPoolSize(len(systemCertPool.Subjects()))) + + return systemCertPool, nil +} diff --git a/pkg/tls/certpool_test.go b/pkg/tls/certpool_test.go new file mode 100644 index 0000000..49cd52e --- /dev/null +++ b/pkg/tls/certpool_test.go @@ -0,0 +1,427 @@ +/* +Copyright SecureKey Technologies Inc. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package tls // nolint:testpackage // references internal implementation details + +import ( + "crypto/x509" + "encoding/pem" + "errors" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + tlsCaOrg1 = `-----BEGIN CERTIFICATE----- +MIICSDCCAe+gAwIBAgIQVy95bDHyGiHPiW/hN7iCEzAKBggqhkjOPQQDAjB2MQsw +CQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZy +YW5jaXNjbzEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEfMB0GA1UEAxMWdGxz +Y2Eub3JnMS5leGFtcGxlLmNvbTAeFw0xODA3MjUxNDQxMjJaFw0yODA3MjIxNDQx +MjJaMHYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH +Ew1TYW4gRnJhbmNpc2NvMRkwFwYDVQQKExBvcmcxLmV4YW1wbGUuY29tMR8wHQYD +VQQDExZ0bHNjYS5vcmcxLmV4YW1wbGUuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0D +AQcDQgAEMl8XK0Rpr514HXVut0MS/PX07l7gWeXGCQkl8T8LBuuSjGEkgSIuOwpf +VqQv4TwXH0A8zIBrtxY2/W3/ERhhC6NfMF0wDgYDVR0PAQH/BAQDAgGmMA8GA1Ud +JQQIMAYGBFUdJQAwDwYDVR0TAQH/BAUwAwEB/zApBgNVHQ4EIgQg+tqYPgAj39pQ +2EH0hxR4SbPOmDRCmwiDsaVIj7tXIFYwCgYIKoZIzj0EAwIDRwAwRAIgUJVxM/57 +1WMfcy56D2zw6g9APP5Z3g+Qg/Y5cScstkgCIBj0JVuemNxiQWdXZ/Qhc6sh4m5d +ngzYatfQtNv3/+4V +-----END CERTIFICATE-----` + + tlsCaOrg2 = `-----BEGIN CERTIFICATE----- +MIICSDCCAe+gAwIBAgIQRAmchEVD9462610qy8BdfDAKBggqhkjOPQQDAjB2MQsw +CQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZy +YW5jaXNjbzEZMBcGA1UEChMQb3JnMi5leGFtcGxlLmNvbTEfMB0GA1UEAxMWdGxz +Y2Eub3JnMi5leGFtcGxlLmNvbTAeFw0xODA3MjUxNDQxMjJaFw0yODA3MjIxNDQx +MjJaMHYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH +Ew1TYW4gRnJhbmNpc2NvMRkwFwYDVQQKExBvcmcyLmV4YW1wbGUuY29tMR8wHQYD +VQQDExZ0bHNjYS5vcmcyLmV4YW1wbGUuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0D +AQcDQgAErO2jmz8uCcrVAghsV0CqWCbp55apZ+4Sww01eYswbeNWsFkXLeUxoDYW +24ClPai2+hXe8djlV/0J9dQN9Lb05aNfMF0wDgYDVR0PAQH/BAQDAgGmMA8GA1Ud +JQQIMAYGBFUdJQAwDwYDVR0TAQH/BAUwAwEB/zApBgNVHQ4EIgQgV28IES/J0+fq +SZlYwCgztWVcEH4gwOvZw3g3y5J194wwCgYIKoZIzj0EAwIDRwAwRAIgYEcFrfpI +gd8ZaCY75B07c87C1FkMJqom3TrdyLbb39kCIHS9zZg6t/W/rmSG6rJlXxqS3RRh +10Y4jiCH6so41N9w +-----END CERTIFICATE-----` + + tlsOrdererCert = `-----BEGIN CERTIFICATE----- +MIICNjCCAdygAwIBAgIRAO47NS1d5RtzwWFIUwWpWT0wCgYIKoZIzj0EAwIwbDEL +MAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDVNhbiBG +cmFuY2lzY28xFDASBgNVBAoTC2V4YW1wbGUuY29tMRowGAYDVQQDExF0bHNjYS5l +eGFtcGxlLmNvbTAeFw0xODA3MjUxNDQxMjJaFw0yODA3MjIxNDQxMjJaMGwxCzAJ +BgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1TYW4gRnJh +bmNpc2NvMRQwEgYDVQQKEwtleGFtcGxlLmNvbTEaMBgGA1UEAxMRdGxzY2EuZXhh +bXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQEQRujXijEID810f9Y +09zBcqucKcz8G4nRgsbfuLZiLgy3Cq/TJsdgjAIvfqR56KtQaupS6tU8xPtLFIr5 +Vr6oo18wXTAOBgNVHQ8BAf8EBAMCAaYwDwYDVR0lBAgwBgYEVR0lADAPBgNVHRMB +Af8EBTADAQH/MCkGA1UdDgQiBCCKGZZgFT855X2eDdYagwYvZB8rkWu7xMsRGvvm +bworhDAKBggqhkjOPQQDAgNIADBFAiEAnmp7VjUxVbfrKRXfpW3X2O27doYxb1Z9 +xjm288m0ljoCID1CMTrMDZn8M/YYpPrw9WkS3n2clykUQeMxeMN8uUCj +-----END CERTIFICATE-----` +) + +func TestTLSCAConfigWithMultipleCerts(t *testing.T) { + // prepare 3 certs + certOrg1, err := getCertFromPEMBytes([]byte(tlsCaOrg1)) + assert.Nil(t, err) + assert.NotNil(t, certOrg1) + + certOrg2, err := getCertFromPEMBytes([]byte(tlsCaOrg2)) + assert.Nil(t, err) + assert.NotNil(t, certOrg2) + + certOrderer, err := getCertFromPEMBytes([]byte(tlsOrdererCert)) + assert.Nil(t, err) + assert.NotNil(t, certOrderer) + + // number of subjects in system cert pool + c, err := loadSystemCertPool(true) + assert.Nil(t, err) + + numberOfSubjects := len(c.Subjects()) //nolint:staticcheck + + // create certpool instance + tlsCertPool, err := NewCertPool(true) + assert.Nil(t, err) + + // empty cert pool with just system cert pool certs + pool, err := tlsCertPool.Get() + assert.Nil(t, err) + verifyCertPoolInstance(t, pool, tlsCertPool, 0, 0, 0, numberOfSubjects, 0) + + // add 2 certs + tlsCertPool.Add(certOrderer, certOrg1) + verifyCertPoolInstance(t, pool, tlsCertPool, 0, 2, 2, numberOfSubjects, 1) + pool, err = tlsCertPool.Get() + assert.Nil(t, err) + verifyCertPoolInstance(t, pool, tlsCertPool, 2, 2, 2, numberOfSubjects, 0) + + // add 1 existing cert, queue should be unchanged and dirty flag should be off + tlsCertPool.Add(certOrg1) + verifyCertPoolInstance(t, pool, tlsCertPool, 2, 2, 2, numberOfSubjects, 0) + pool, err = tlsCertPool.Get() + assert.Nil(t, err) + verifyCertPoolInstance(t, pool, tlsCertPool, 2, 2, 2, numberOfSubjects, 0) + + // try again, add 1 existing cert, queue should be unchanged and dirty flag should be off + tlsCertPool.Add(certOrg1) + verifyCertPoolInstance(t, pool, tlsCertPool, 2, 2, 2, numberOfSubjects, 0) + pool, err = tlsCertPool.Get() + assert.Nil(t, err) + verifyCertPoolInstance(t, pool, tlsCertPool, 2, 2, 2, numberOfSubjects, 0) + + // add 2 existing certs, queue should be unchanged and dirty flag should be off + tlsCertPool.Add(certOrderer, certOrg1) + verifyCertPoolInstance(t, pool, tlsCertPool, 2, 2, 2, + numberOfSubjects, 0) + + pool, err = tlsCertPool.Get() + assert.Nil(t, err) + verifyCertPoolInstance(t, pool, tlsCertPool, 2, 2, 2, + numberOfSubjects, 0) + + // add 3 certs, (2 existing + 1 new), queue should have one extra cert and dirty flag should be on + tlsCertPool.Add(certOrderer, certOrg1, certOrg2) + verifyCertPoolInstance(t, pool, tlsCertPool, 2, 3, 3, + numberOfSubjects, 1) + + pool, err = tlsCertPool.Get() + assert.Nil(t, err) + verifyCertPoolInstance(t, pool, tlsCertPool, 3, 3, 3, + numberOfSubjects, 0) + + // add all 3 existing certs, queue should be unchanged and dirty flag should be off + tlsCertPool.Add(certOrderer, certOrg1, certOrg2) + verifyCertPoolInstance(t, pool, tlsCertPool, 3, 3, 3, + numberOfSubjects, 0) + + pool, err = tlsCertPool.Get() + assert.Nil(t, err) + verifyCertPoolInstance(t, pool, tlsCertPool, 3, 3, 3, + numberOfSubjects, 0) +} + +func verifyCertPoolInstance(t *testing.T, pool *x509.CertPool, tlsCertPool *CertPool, numberOfCertsInPool, + numberOfCerts, numberOfCertsByName, numberOfSubjects int, dirty int32) { + t.Helper() + + assert.NotNil(t, tlsCertPool) + assert.Equal(t, dirty, tlsCertPool.dirty) + assert.Equal(t, numberOfCerts, len(tlsCertPool.certs)) + assert.Equal(t, numberOfCertsByName, len(tlsCertPool.certsByName)) + assert.Equal(t, numberOfSubjects+numberOfCertsInPool, len(pool.Subjects())) //nolint:staticcheck +} + +func TestAddingDuplicateCertsToPool(t *testing.T) { + // prepare 3 certs + certOrg1, err := getCertFromPEMBytes([]byte(tlsCaOrg1)) + assert.Nil(t, err) + assert.NotNil(t, certOrg1) + + certOrg2, err := getCertFromPEMBytes([]byte(tlsCaOrg2)) + assert.Nil(t, err) + assert.NotNil(t, certOrg2) + + certOrderer, err := getCertFromPEMBytes([]byte(tlsOrdererCert)) + assert.Nil(t, err) + assert.NotNil(t, certOrderer) + + // number of subjects in system cert pool + c, err := loadSystemCertPool(true) + assert.Nil(t, err) + + numberOfSubjects := len(c.Subjects()) //nolint:staticcheck + + // create certpool instance + tlsCertPool, err := NewCertPool(true) + assert.Nil(t, err) + + // empty cert pool with just system cert pool certs + pool, err := tlsCertPool.Get() + assert.Nil(t, err) + verifyCertPoolInstance(t, pool, tlsCertPool, 0, 0, 0, numberOfSubjects, 0) + + // add multiple certs with duplicate + tlsCertPool.Add(certOrderer, certOrg1, certOrderer, certOrg1, certOrg1, certOrg1, certOrderer, certOrderer) + verifyCertPoolInstance(t, pool, tlsCertPool, 0, 2, 2, numberOfSubjects, 1) + pool, err = tlsCertPool.Get() + assert.Nil(t, err) + verifyCertPoolInstance(t, pool, tlsCertPool, 2, 2, 2, numberOfSubjects, 0) + + // add multiple certs with duplicate + tlsCertPool.Add(certOrderer, certOrg1, certOrderer, certOrg1, certOrg2, certOrg2, certOrg2, certOrderer, certOrderer) + verifyCertPoolInstance(t, pool, tlsCertPool, 2, 3, 3, numberOfSubjects, 1) + pool, err = tlsCertPool.Get() + assert.Nil(t, err) + verifyCertPoolInstance(t, pool, tlsCertPool, 3, 3, 3, numberOfSubjects, 0) +} + +func TestRemoveDuplicatesCerts(t *testing.T) { + // prepare 3 certs + certOrg1, err := getCertFromPEMBytes([]byte(tlsCaOrg1)) + assert.Nil(t, err) + assert.NotNil(t, certOrg1) + + certOrg2, err := getCertFromPEMBytes([]byte(tlsCaOrg2)) + assert.Nil(t, err) + assert.NotNil(t, certOrg2) + + certOrderer, err := getCertFromPEMBytes([]byte(tlsOrdererCert)) + assert.Nil(t, err) + assert.NotNil(t, certOrderer) + + certs := removeDuplicates(certOrg1, certOrg2, certOrg1, certOrg1, certOrg1, certOrg1, certOrderer) + assert.Equal(t, 3, len(certs)) + + var hasCertOrg1, hasCertOrg2, hasOrdCert bool + + for _, c := range certs { + switch c.Subject.CommonName { + case "tlsca.org1.example.com": + hasCertOrg1 = true + case "tlsca.org2.example.com": + hasCertOrg2 = true + case "tlsca.example.com": + hasOrdCert = true + } + } + + assert.True(t, hasCertOrg1) + assert.True(t, hasCertOrg2) + assert.True(t, hasOrdCert) +} + +func TestTLSCAConfig(t *testing.T) { + goodCert := &x509.Certificate{ + RawSubject: []byte("Good header"), + Raw: []byte("Good cert"), + } + + tlsCertPool, err := NewCertPool(true) + require.NoError(t, err) + + tlsCertPool.Add(goodCert) + _, err = tlsCertPool.Get() + require.NoError(t, err) + assert.NotNil(t, tlsCertPool.certsByName) + + originalLength := len(tlsCertPool.certs) + // try again with same cert + tlsCertPool.Add(goodCert) + _, err = tlsCertPool.Get() + assert.NoError(t, err, "TLS CA cert pool fetch failed") + assert.False(t, len(tlsCertPool.certs) > originalLength, "number of certs in cert list shouldn't accept duplicates") + + // test with system cert pool disabled + tlsCertPool, err = NewCertPool(false) + require.NoError(t, err) + + tlsCertPool.Add(goodCert) + cPool, err := tlsCertPool.Get() + require.NoError(t, err) + assert.Len(t, tlsCertPool.certs, 1) + assert.Len(t, cPool.Subjects(), 1) //nolint:staticcheck +} + +func TestTLSCAPoolManyCerts(t *testing.T) { + size := 50 + goodCert := &x509.Certificate{ + RawSubject: []byte("Good header"), + Raw: []byte("Good cert"), + } + + tlsCertPool, err := NewCertPool(true) + require.NoError(t, err) + + tlsCertPool.Add(goodCert) + _, err = tlsCertPool.Get() + require.NoError(t, err) + + pool, err := tlsCertPool.Get() + assert.NoError(t, err) + + originalLen := len(pool.Subjects()) //nolint:staticcheck + + certs := createNCerts(size) + tlsCertPool.Add(certs[0]) + pool, err = tlsCertPool.Get() + assert.NoError(t, err) + assert.Len(t, pool.Subjects(), originalLen+1) //nolint:staticcheck + + tlsCertPool.Add(certs...) + pool, err = tlsCertPool.Get() + assert.NoError(t, err) + assert.Len(t, pool.Subjects(), originalLen+size) //nolint:staticcheck +} + +func TestConcurrent(t *testing.T) { + concurrency := 1000 + certs := createNCerts(concurrency) + + tlsCertPool, err := NewCertPool(false) + require.NoError(t, err) + + systemCerts := len(tlsCertPool.certPool.Subjects()) //nolint:staticcheck + + writeDone := make(chan bool) + readDone := make(chan bool) + + for i := 0; i < concurrency; i++ { + go func(c *x509.Certificate) { + tlsCertPool.Add(c) + _, errGet := tlsCertPool.Get() + assert.NoError(t, errGet) + writeDone <- true + }(certs[i]) + + go func() { + _, errGet := tlsCertPool.Get() + assert.NoError(t, errGet) + readDone <- true + }() + } + + for i := 0; i < concurrency; i++ { + select { + case b := <-writeDone: + assert.True(t, b) + case <-time.After(time.Second * 10): + t.Fatalf("Timed out waiting for write %d", i) + } + + select { + case b := <-readDone: + assert.True(t, b) + case <-time.After(time.Second * 10): + t.Fatalf("Timed out waiting for read %d", i) + } + } + + certPool, err := tlsCertPool.Get() + assert.Len(t, tlsCertPool.certs, concurrency) + require.NoError(t, err) + assert.Len(t, certPool.Subjects(), concurrency+systemCerts) //nolint:staticcheck +} + +func createNCerts(n int) []*x509.Certificate { + var certs []*x509.Certificate + + for i := 0; i < n; i++ { + cert := &x509.Certificate{ + RawSubject: []byte(strconv.Itoa(i)), + Raw: []byte(strconv.Itoa(i)), + } + certs = append(certs, cert) + } + + return certs +} + +func BenchmarkTLSCertPool(b *testing.B) { + tlsCertPool, err := NewCertPool(true) + require.NoError(b, err) + + for n := 0; n < b.N; n++ { + _, err := tlsCertPool.Get() + require.NoError(b, err) + } +} + +func BenchmarkTLSCertPoolSameCert(b *testing.B) { + goodCert := &x509.Certificate{ + RawSubject: []byte("Good header"), + Raw: []byte("Good cert"), + } + + tlsCertPool, err := NewCertPool(true) + require.NoError(b, err) + + for n := 0; n < b.N; n++ { + tlsCertPool.Add(goodCert) + _, err = tlsCertPool.Get() + require.NoError(b, err) + } +} + +func BenchmarkTLSCertPoolDifferentCert(b *testing.B) { + tlsCertPool, err := NewCertPool(true) + require.NoError(b, err) + + certs := createNCerts(b.N) + + for n := 0; n < b.N; n++ { + tlsCertPool.Add(certs[n]) + _, err = tlsCertPool.Get() + require.NoError(b, err) + } +} + +func getCertFromPEMBytes(pemCerts []byte) (*x509.Certificate, error) { + for len(pemCerts) > 0 { + var block *pem.Block + block, pemCerts = pem.Decode(pemCerts) + + if block == nil { + break + } + + if block.Type != "CERTIFICATE" || len(block.Headers) != 0 { + continue + } + + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + continue + } + + return cert, nil + } + + return nil, errors.New("empty cert bytes provided") +} diff --git a/pkg/utils/cmd/util.go b/pkg/utils/cmd/util.go new file mode 100644 index 0000000..2a79595 --- /dev/null +++ b/pkg/utils/cmd/util.go @@ -0,0 +1,152 @@ +/* +Copyright SecureKey Technologies Inc. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package cmd + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" +) + +// GetUserSetOptionalVarFromString returns values either command line flag or environment variable. +func GetUserSetOptionalVarFromString(cmd *cobra.Command, flagName, envKey string) string { + //nolint // the error will not happen for optional var + v, _ := GetUserSetVarFromString(cmd, flagName, envKey, true) + + return v +} + +// GetUserSetVarFromString returns values either command line flag or environment variable. +func GetUserSetVarFromString(cmd *cobra.Command, flagName, envKey string, isOptional bool) (string, error) { + if cmd.Flags().Changed(flagName) { + value, err := cmd.Flags().GetString(flagName) + if err != nil { + return "", fmt.Errorf(flagName+" flag not found: %s", err) + } + + if value == "" { + return "", fmt.Errorf("%s value is empty", flagName) + } + + return value, nil + } + + value, isSet := os.LookupEnv(envKey) + + if isOptional || isSet { + if !isOptional && value == "" { + return "", fmt.Errorf("%s value is empty", envKey) + } + + return value, nil + } + + return "", errors.New("Neither " + flagName + " (command line flag) nor " + envKey + + " (environment variable) have been set.") +} + +// GetUserSetOptionalVarFromArrayString returns the variables set via either command line flag or environment variable. +// If both are set, then the command line flag takes precedence. +// For the command line flag, the variables must be set using repeated flags (e.g. --flagName value1 --flagName value2). +// For the environment variable, the variables are parsed as comma-separated-values (CSV) and returned as a slice. +// The command line flag must be set as a StringArray. +// If the variable isn't set, then an empty or nil slice will be returned. +func GetUserSetOptionalVarFromArrayString(cmd *cobra.Command, flagName, envKey string) []string { + //nolint // reason the error will not happen for optional var + v, _ := GetUserSetVarFromArrayString(cmd, flagName, envKey, true) + + return v +} + +// GetUserSetVarFromArrayString returns the variables set via either command line flag or environment variable. +// If both are set, then the command line flag takes precedence. +// For the command line flag, the variables must be set using repeated flags (e.g. --flagName value1 --flagName value2). +// For the environment variable, the variables are parsed as comma-separated-values (CSV) and returned as a slice. +// The command line flag must be set as a StringArray. +// If the variable isn't set, then an error will be returned. +func GetUserSetVarFromArrayString(cmd *cobra.Command, flagName, envKey string, isOptional bool) ([]string, error) { + if cmd.Flags().Changed(flagName) { + value, err := cmd.Flags().GetStringArray(flagName) + if err != nil { + return nil, fmt.Errorf(flagName+" flag not found: %s", err) + } + + if len(value) == 0 { + return nil, fmt.Errorf("%s value is empty", flagName) + } + + return value, nil + } + + value, isSet := os.LookupEnv(envKey) + + if isOptional || isSet { + if !isOptional && value == "" { + return nil, fmt.Errorf("%s value is empty", envKey) + } + + if value == "" { + return []string{}, nil + } + + return strings.Split(value, ","), nil + } + + return nil, errors.New("Neither " + flagName + " (command line flag) nor " + envKey + + " (environment variable) have been set.") +} + +// GetUserSetOptionalCSVVar returns the variables set via either command line flag or environment variable. +// If both are set, then the command line flag takes precedence. +// The variables are parsed as comma-separated-values (CSV) and returned as a slice. +// The command line flag must be set as a StringSlice. +// If the variable isn't set, then a nil slice will be returned. +func GetUserSetOptionalCSVVar(cmd *cobra.Command, flagName, envKey string) []string { + //nolint // For an optional variable, no error will happen (or we don't care about the error) + v, _ := GetUserSetCSVVar(cmd, flagName, envKey, true) + + return v +} + +// GetUserSetCSVVar returns the variables set via either command line flag or environment variable. +// If both are set, then the command line flag takes precedence. +// The variables are parsed as comma-separated-values (CSV) and returned as a slice. +// The command line flag must be set as a StringSlice. +// If the variable isn't set, then an error will be returned. +func GetUserSetCSVVar(cmd *cobra.Command, flagName, envKey string, isOptional bool) ([]string, error) { + if cmd.Flags().Changed(flagName) { + value, err := cmd.Flags().GetStringSlice(flagName) + if err != nil { + return nil, fmt.Errorf(flagName+" flag not found: %s", err) + } + + if len(value) == 0 { + return nil, fmt.Errorf("%s value is empty", flagName) + } + + return value, nil + } + + value, isSet := os.LookupEnv(envKey) + + if isOptional || isSet { + if !isOptional && value == "" { + return nil, fmt.Errorf("%s value is empty", envKey) + } + + if value == "" { + return nil, nil + } + + return strings.Split(value, ","), nil + } + + return nil, errors.New("Neither " + flagName + " (command line flag) nor " + envKey + + " (environment variable) have been set.") +} diff --git a/pkg/utils/cmd/util_test.go b/pkg/utils/cmd/util_test.go new file mode 100644 index 0000000..8920200 --- /dev/null +++ b/pkg/utils/cmd/util_test.go @@ -0,0 +1,266 @@ +/* +Copyright SecureKey Technologies Inc. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package cmd_test + +import ( + "os" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" + + "github.com/trustbloc/cmdutil-go/pkg/utils/cmd" +) + +const ( + flagName = "host-url" + envKey = "TEST_HOST_URL" + testHostURLVar = "localhost:8080" +) + +func TestGetUserSetVarFromStringNegative(t *testing.T) { + os.Clearenv() + + command := &cobra.Command{ + Use: "start", + Short: "short usage", + Long: "long usage", + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + // test missing both command line argument and environment vars + env, err := cmd.GetUserSetVarFromString(command, flagName, envKey, false) + require.Error(t, err) + require.Empty(t, env) + require.Contains(t, err.Error(), "TEST_HOST_URL (environment variable) have been set.") + + // test env var is empty + t.Setenv(envKey, "") + + env, err = cmd.GetUserSetVarFromString(command, flagName, envKey, false) + require.Error(t, err) + require.Contains(t, err.Error(), "TEST_HOST_URL value is empty") + require.Empty(t, env) + + // test arg is empty + command.Flags().StringP(flagName, "", "initial", "") + args := []string{"--" + flagName, ""} + command.SetArgs(args) + err = command.Execute() + require.NoError(t, err) + + env, err = cmd.GetUserSetVarFromString(command, flagName, envKey, false) + require.Error(t, err) + require.Contains(t, err.Error(), "host-url value is empty") + require.Empty(t, env) +} + +func TestGetUserSetVarFromArrayStringNegative(t *testing.T) { + os.Clearenv() + + command := &cobra.Command{ + Use: "start", + Short: "short usage", + Long: "long usage", + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + // test missing both command line argument and environment vars + env, err := cmd.GetUserSetVarFromArrayString(command, flagName, envKey, false) + require.Error(t, err) + require.Empty(t, env) + require.Contains(t, err.Error(), "TEST_HOST_URL (environment variable) have been set.") + + // test env var is empty + t.Setenv(envKey, "") + + env, err = cmd.GetUserSetVarFromArrayString(command, flagName, envKey, false) + require.Error(t, err) + require.Contains(t, err.Error(), "TEST_HOST_URL value is empty") + require.Empty(t, env) + + // try again, but this time make it optional + env, err = cmd.GetUserSetVarFromArrayString(command, flagName, envKey, true) + require.NoError(t, err) + require.Empty(t, env) + + // test arg is empty + command.Flags().StringArrayP(flagName, "", []string{}, "") + args := []string{"--" + flagName, ""} + command.SetArgs(args) + err = command.Execute() + require.NoError(t, err) + + env, err = cmd.GetUserSetVarFromArrayString(command, flagName, envKey, false) + require.Error(t, err) + require.Contains(t, err.Error(), "host-url value is empty") + require.Empty(t, env) +} + +func TestGetUserSetCSVVarNegative(t *testing.T) { + os.Clearenv() + + command := &cobra.Command{ + Use: "start", + Short: "short usage", + Long: "long usage", + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + // test missing both command line argument and environment vars + env, err := cmd.GetUserSetCSVVar(command, flagName, envKey, false) + require.Error(t, err) + require.Empty(t, env) + require.Contains(t, err.Error(), "TEST_HOST_URL (environment variable) have been set.") + + // test env var is empty + t.Setenv(envKey, "") + + env, err = cmd.GetUserSetCSVVar(command, flagName, envKey, false) + require.Error(t, err) + require.Contains(t, err.Error(), "TEST_HOST_URL value is empty") + require.Empty(t, env) + + // try again, but this time make it optional + env, err = cmd.GetUserSetCSVVar(command, flagName, envKey, true) + require.NoError(t, err) + require.Empty(t, env) + + // test arg is empty + command.Flags().StringSliceP(flagName, "", []string{}, "") + args := []string{"--" + flagName, ""} + command.SetArgs(args) + err = command.Execute() + require.NoError(t, err) + + env, err = cmd.GetUserSetCSVVar(command, flagName, envKey, false) + require.Error(t, err) + require.Contains(t, err.Error(), "host-url value is empty") + require.Empty(t, env) +} + +func TestGetUserSetVarFromString(t *testing.T) { + os.Clearenv() + + command := &cobra.Command{ + Use: "start", + Short: "short usage", + Long: "long usage", + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + // test env var is set + t.Setenv(envKey, testHostURLVar) + + // test resolution via environment variable + env, err := cmd.GetUserSetVarFromString(command, flagName, envKey, false) + require.NoError(t, err) + require.Equal(t, testHostURLVar, env) + + // set command line arguments + command.Flags().StringP(flagName, "", "initial", "") + args := []string{"--" + flagName, "other"} + command.SetArgs(args) + err = command.Execute() + require.NoError(t, err) + + err = os.Unsetenv(envKey) + require.NoError(t, err) + + // test resolution via command line argument - no environment variable set + env, err = cmd.GetUserSetVarFromString(command, flagName, "", false) + require.NoError(t, err) + require.Equal(t, "other", env) + + env = cmd.GetUserSetOptionalVarFromString(command, flagName, "") + require.Equal(t, "other", env) +} + +func TestGetUserSetVarFromArrayString(t *testing.T) { + os.Clearenv() + + command := &cobra.Command{ + Use: "start", + Short: "short usage", + Long: "long usage", + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + // test env var is set + t.Setenv(envKey, testHostURLVar) + + // test resolution via environment variable + env, err := cmd.GetUserSetVarFromArrayString(command, flagName, envKey, false) + require.NoError(t, err) + require.Equal(t, []string{testHostURLVar}, env) + + // set command line arguments + command.Flags().StringArrayP(flagName, "", []string{}, "") + args := []string{"--" + flagName, "other", "--" + flagName, "other1"} + command.SetArgs(args) + err = command.Execute() + require.NoError(t, err) + + err = os.Unsetenv(envKey) + require.NoError(t, err) + + // test resolution via command line argument - no environment variable set + env, err = cmd.GetUserSetVarFromArrayString(command, flagName, "", false) + require.NoError(t, err) + require.Equal(t, []string{"other", "other1"}, env) + + env = cmd.GetUserSetOptionalVarFromArrayString(command, flagName, "") + require.Equal(t, []string{"other", "other1"}, env) +} + +func TestGetUserSetCSVVar(t *testing.T) { + os.Clearenv() + + command := &cobra.Command{ + Use: "start", + Short: "short usage", + Long: "long usage", + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + // test env var is set + t.Setenv(envKey, testHostURLVar) + + // test resolution via environment variable + env, err := cmd.GetUserSetCSVVar(command, flagName, envKey, false) + require.NoError(t, err) + require.Equal(t, []string{testHostURLVar}, env) + + // set command line arguments + command.Flags().StringSliceP(flagName, "", []string{}, "") + args := []string{"--" + flagName, "other,other1"} + command.SetArgs(args) + err = command.Execute() + require.NoError(t, err) + + err = os.Unsetenv(envKey) + require.NoError(t, err) + + // test resolution via command line argument - no environment variable set + env, err = cmd.GetUserSetCSVVar(command, flagName, "", false) + require.NoError(t, err) + require.Equal(t, []string{"other", "other1"}, env) + + env = cmd.GetUserSetOptionalCSVVar(command, flagName, "") + require.Equal(t, []string{"other", "other1"}, env) +} diff --git a/pkg/utils/tls/util.go b/pkg/utils/tls/util.go new file mode 100644 index 0000000..07b27b9 --- /dev/null +++ b/pkg/utils/tls/util.go @@ -0,0 +1,45 @@ +/* +Copyright SecureKey Technologies Inc. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package tls + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "os" + "path" + + commontls "github.com/trustbloc/cmdutil-go/pkg/tls" +) + +// GetCertPool get cert pool. +func GetCertPool(useSystemCertPool bool, tlsCACerts []string) (*x509.CertPool, error) { + certPool, err := commontls.NewCertPool(useSystemCertPool) + if err != nil { + return nil, fmt.Errorf("failed to create new cert pool: %w", err) + } + + for _, v := range tlsCACerts { + bytes, errRead := os.ReadFile(path.Clean(v)) + if errRead != nil { + return nil, fmt.Errorf("failed to read cert: %w", errRead) + } + + block, _ := pem.Decode(bytes) + if block == nil { + return nil, fmt.Errorf("failed to decode pem") + } + + cert, errParse := x509.ParseCertificate(block.Bytes) + if errParse != nil { + return nil, fmt.Errorf("failed to parse cert: %w", errParse) + } + + certPool.Add(cert) + } + + return certPool.Get() +} diff --git a/pkg/utils/tls/util_test.go b/pkg/utils/tls/util_test.go new file mode 100644 index 0000000..dc97a88 --- /dev/null +++ b/pkg/utils/tls/util_test.go @@ -0,0 +1,69 @@ +/* +Copyright SecureKey Technologies Inc. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package tls_test + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/trustbloc/cmdutil-go/pkg/utils/tls" +) + +const ( + tlsCaOrg1 = `-----BEGIN CERTIFICATE----- +MIICSDCCAe+gAwIBAgIQVy95bDHyGiHPiW/hN7iCEzAKBggqhkjOPQQDAjB2MQsw +CQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZy +YW5jaXNjbzEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEfMB0GA1UEAxMWdGxz +Y2Eub3JnMS5leGFtcGxlLmNvbTAeFw0xODA3MjUxNDQxMjJaFw0yODA3MjIxNDQx +MjJaMHYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH +Ew1TYW4gRnJhbmNpc2NvMRkwFwYDVQQKExBvcmcxLmV4YW1wbGUuY29tMR8wHQYD +VQQDExZ0bHNjYS5vcmcxLmV4YW1wbGUuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0D +AQcDQgAEMl8XK0Rpr514HXVut0MS/PX07l7gWeXGCQkl8T8LBuuSjGEkgSIuOwpf +VqQv4TwXH0A8zIBrtxY2/W3/ERhhC6NfMF0wDgYDVR0PAQH/BAQDAgGmMA8GA1Ud +JQQIMAYGBFUdJQAwDwYDVR0TAQH/BAUwAwEB/zApBgNVHQ4EIgQg+tqYPgAj39pQ +2EH0hxR4SbPOmDRCmwiDsaVIj7tXIFYwCgYIKoZIzj0EAwIDRwAwRAIgUJVxM/57 +1WMfcy56D2zw6g9APP5Z3g+Qg/Y5cScstkgCIBj0JVuemNxiQWdXZ/Qhc6sh4m5d +ngzYatfQtNv3/+4V +-----END CERTIFICATE-----` +) + +func TestGetCertPool(t *testing.T) { + t.Run("test wrong file path", func(t *testing.T) { + certPool, err := tls.GetCertPool(false, []string{"wrongLocation"}) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to read cert") + require.Nil(t, certPool) + }) + + t.Run("test error from decode pem", func(t *testing.T) { + file, err := os.CreateTemp("", "file") + require.NoError(t, err) + + _, err = file.Write([]byte("data")) + require.NoError(t, err) + + defer func() { require.NoError(t, os.Remove(file.Name())) }() + certPool, err := tls.GetCertPool(false, []string{file.Name()}) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to decode pem") + require.Nil(t, certPool) + }) + + t.Run("test error from success", func(t *testing.T) { + file, err := os.CreateTemp("", "file") + require.NoError(t, err) + + _, err = file.Write([]byte(tlsCaOrg1)) + require.NoError(t, err) + + defer func() { require.NoError(t, os.Remove(file.Name())) }() + certPool, err := tls.GetCertPool(false, []string{file.Name()}) + require.NoError(t, err) + require.NotNil(t, certPool) + }) +} diff --git a/scripts/check_license.sh b/scripts/check_license.sh new file mode 100755 index 0000000..53b3d5f --- /dev/null +++ b/scripts/check_license.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Copyright IBM Corp, SecureKey Technologies Inc. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# + +echo "Running $0" + +function filterExcludedFiles { + CHECK=`echo "$CHECK" | grep -v .png$ | grep -v .rst$ | grep -v ^.git/ \ + | grep -v .pem$ | grep -v .block$ | grep -v .tx$ | grep -v ^LICENSE$ | grep -v _sk$ \ + | grep -v .key$ | grep -v .crt$ | grep -v \\.gen.go$ | grep -v \\.json$ | grep -v \\.jsonld | grep -v Gopkg.lock$ \ + | grep -v .md$ | grep -v ^vendor/ | grep -v ^build/ | grep -v .pb.go$ | grep -v ci.properties$ \ + | grep -v go.sum$ | grep -v openapi/ | grep -v \\.jwt | sort -u` +} + +CHECK=$(git diff --name-only --diff-filter=ACMRTUXB HEAD) +REMOTE_REF=$(git log -1 --pretty=format:"%d" | grep '[(].*\/' | wc -l) + +# If CHECK is empty then there is no working directory changes: fallback to last two commits. +# Else if REMOTE_REF=0 then working copy commits are even with remote: only use the working copy changes. +# Otherwise assume that the change is amending the previous commit: use both last two commit and working copy changes. +if [[ -z "${CHECK}" ]] || [[ "${REMOTE_REF}" -eq 0 ]]; then + if [[ ! -z "${CHECK}" ]]; then + echo "Examining last commit and working directory changes" + CHECK+=$'\n' + else + echo "Examining last commit changes" + fi + + LAST_COMMITS=($(git log -2 --pretty=format:"%h")) + CHECK+=$(git diff-tree --no-commit-id --name-only --diff-filter=ACMRTUXB -r ${LAST_COMMITS[1]} ${LAST_COMMITS[0]}) +else + echo "Examining working directory changes" +fi + +filterExcludedFiles + +if [[ -z "$CHECK" ]]; then + echo "All files are excluded from having license headers" + exit 0 +fi + +missing=`echo "$CHECK" | xargs ls -d 2>/dev/null | xargs grep -L "SPDX-License-Identifier"` +if [[ -z "$missing" ]]; then + echo "All files have SPDX-License-Identifier headers" + exit 0 +fi +echo "The following files are missing SPDX-License-Identifier headers:" +echo "$missing" +echo +echo "Please replace the Apache license header comment text with:" +echo "SPDX-License-Identifier: Apache-2.0" + +echo +echo "Checking committed files for traditional Apache License headers ..." +missing=`echo "$missing" | xargs ls -d 2>/dev/null | xargs grep -L "http://www.apache.org/licenses/LICENSE-2.0"` +if [[ -z "$missing" ]]; then + echo "All remaining files have Apache 2.0 headers" + exit 0 +fi +echo "The following files are missing traditional Apache 2.0 headers:" +echo "$missing" +echo "Fatal Error - All files must have a license header" +exit 1 diff --git a/scripts/check_lint.sh b/scripts/check_lint.sh new file mode 100755 index 0000000..7c6058e --- /dev/null +++ b/scripts/check_lint.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# +# Copyright SecureKey Technologies Inc. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +echo "Running $0" + +DOCKER_CMD=${DOCKER_CMD:-docker} +GOLANGCI_LINT_IMAGE="golangci/golangci-lint:v1.50.1" + +if [ ! $(command -v ${DOCKER_CMD}) ]; then + exit 0 +fi + +${DOCKER_CMD} run --rm -e GOPROXY=${GOPROXY} -v $(pwd):/opt/workspace -w /opt/workspace ${GOLANGCI_LINT_IMAGE} golangci-lint run --timeout 5m diff --git a/scripts/check_unit.sh b/scripts/check_unit.sh new file mode 100755 index 0000000..a3e929d --- /dev/null +++ b/scripts/check_unit.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright SecureKey Technologies Inc. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +set -e + +echo "Running $0" + +pwd=`pwd` +touch "$pwd"/coverage.out + +amend_coverage_file () { +if [ -f profile.out ]; then + cat profile.out | grep -v ".gen.go" >> "$pwd"/coverage.out + rm profile.out +fi +} + +# Running cmdutil-go unit tests +echo "cmdutil-go unit tests..." +PKGS=`go list github.com/trustbloc/cmdutil-go/... 2> /dev/null | \ + grep -v /mocks` +go test $PKGS -count=1 -race -coverprofile=profile.out -covermode=atomic -timeout=10m +amend_coverage_file +echo "... done unit tests" + +cd "$pwd"