From 9597251f7d5c9da1a834d8818db2b33f934e0f61 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Sat, 16 Dec 2023 21:10:55 +0100 Subject: [PATCH 1/2] test: refactoring e2e & adding tests archivistactl This commit improves the archivistactl end-to-end (e2e) tests moving from shell script to testify test suite. The refactoring enables Archivista to have a better view of its test coverage. This improvement also includes tests for other assert levels, such as expected messages and data content, which will avoid issues such as regression. Signed-off-by: Kairo de Araujo --- .github/workflows/pipeline.yml | 4 +- .gitignore | 4 +- Makefile | 14 +- cmd/archivistactl/cmd/e2e_test.go | 212 +++++++++++++++++++++++++ cmd/archivistactl/cmd/retrieve.go | 2 +- cmd/archivistactl/cmd/retrieve_test.go | 94 +++++++++++ cmd/archivistactl/cmd/root_test.go | 45 ++++++ cmd/archivistactl/cmd/search.go | 7 +- cmd/archivistactl/cmd/search_test.go | 70 ++++++++ cmd/archivistactl/cmd/store.go | 2 +- test/{test.sh => deploy-services.sh} | 52 ++++-- test/invalid_payload.attestation.json | 5 + 12 files changed, 486 insertions(+), 25 deletions(-) create mode 100644 cmd/archivistactl/cmd/e2e_test.go create mode 100644 cmd/archivistactl/cmd/retrieve_test.go create mode 100644 cmd/archivistactl/cmd/root_test.go create mode 100644 cmd/archivistactl/cmd/search_test.go rename test/{test.sh => deploy-services.sh} (70%) create mode 100644 test/invalid_payload.attestation.json diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 44356ca7..a1eb5096 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -56,7 +56,9 @@ jobs: pull_request: ${{ github.event_name == 'pull_request' }} step: "e2e" attestations: "github" - command: ./test/test.sh + command: | + make clean + make test release: needs: e2e-tests diff --git a/.gitignore b/.gitignore index d087ddfb..2ef16a64 100644 --- a/.gitignore +++ b/.gitignore @@ -14,13 +14,13 @@ # Dependency directories (remove the comment below to include it) # vendor/ -*.json +./*.json .gitignore .idea/ ## protect against development environment being leaked out *.pem *.nix -./archivist +./archivista ./archivistctl .witness.yaml diff --git a/Makefile b/Makefile index c48646e5..38408ca7 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,19 @@ clean: ## Clean up the dev server .PHONY: test test: ## Run tests - @bash ./test/test.sh + @go test ./... -covermode atomic -coverprofile=cover.out -v + +.PHONY: coverage +coverage: ## Show html coverage + @go tool cover -html=cover.out + + +.PHONY: lint +lint: ## Run linter + @golangci-lint run + @go fmt ./... + @go vet ./... + help: ## Show this help @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/cmd/archivistactl/cmd/e2e_test.go b/cmd/archivistactl/cmd/e2e_test.go new file mode 100644 index 00000000..787c1406 --- /dev/null +++ b/cmd/archivistactl/cmd/e2e_test.go @@ -0,0 +1,212 @@ +// Copyright 2023 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path" + "strings" + "testing" + + "github.com/stretchr/testify/suite" +) + +// Test Suite: E2E +type E2EStoreSuite struct { + suite.Suite +} + +func TestE2EStoreSuite(t *testing.T) { + suite.Run(t, new(E2EStoreSuite)) +} + +// TearDown the container deployment +func (e2e *E2EStoreSuite) TearDownTest() { + e2e.T().Log("Stopping up containers") + path, _ := os.Getwd() + fmt.Print(path) + cmd := exec.Command("bash", "../../../test/deploy-services.sh", "stop") + err := cmd.Start() + if err != nil { + e2e.FailNow(err.Error()) + } + err = cmd.Wait() + if err != nil { + e2e.FailNow(err.Error()) + } +} + +// Run the E2E tests +func (e2e *E2EStoreSuite) Test_E2E() { + + // Define tests for supported dbs + testDBCases := []string{"mysql", "pgsql"} + + // Call the script to deploy the containers for the test cases + for _, testDB := range testDBCases { + cmd := exec.Command("bash", "../../../test/deploy-services.sh", "start-"+testDB) + err := cmd.Start() + if err != nil { + e2e.FailNow(err.Error()) + } + e2e.T().Log("Starting services using DB: " + testDB) + err = cmd.Wait() + if err != nil { + e2e.FailNow(err.Error()) + } + + // define test cases struct + type testCases struct { + name string + attestation string // files are stored in test/ + sha256 string + expectedStore string + gitoidStore string // this value is added during `activistactl store command` + expectedSearch string + expectedError string + expectedRetrieveSub string + } + + // test cases + testTable := []testCases{ + { + name: "valid build attestation", + attestation: "../../../test/build.attestation.json", + sha256: "423da4cff198bbffbe3220ed9510d32ba96698e4b1f654552521d1f541abb6dc", + expectedStore: "stored with gitoid", + expectedSearch: "Collection name: build", + expectedRetrieveSub: "Name: https://witness.dev/attestations/product/v0.1/file:testapp", + }, + { + name: "valid package attestation", + sha256: "10cbf0f3d870934921276f669ab707983113f929784d877f1192f43c581f2070", + attestation: "../../../test/package.attestation.json", + expectedStore: "stored with gitoid", + expectedSearch: "Collection name: package", + expectedRetrieveSub: "Name: https://witness.dev/attestations/git/v0.1/commithash:be20100af602c780deeef50c54f5338662ce917c", + }, + { + name: "duplicated package attestation", + sha256: "10cbf0f3d870934921276f669ab707983113f929784d877f1192f43c581f2070", + attestation: "../../../test/package.attestation.json", + expectedStore: "", + expectedSearch: "Collection name: package", + expectedError: "uplicate", + }, + { + name: "fail attestation", + attestation: "../../../test/fail.attestation.json", + sha256: "5e8c57df8ae58fe9a29b29f9993e2fc3b25bd75eb2754f353880bad4b9ebfdb3", + expectedStore: "stored with gitoid", + expectedSearch: "", + expectedRetrieveSub: "Name: https://witness.dev/attestations/git/v0.1/parenthash:aa35c1f4b1d41c87e139c2d333f09117fd0daf4f", + }, + { + name: "invalid payload attestation", + attestation: "../../../test/invalid_payload.attestation.json", + sha256: "5e8c57df8ae58fe9a29b29f9993e2fc3b25bd75eb2754f353880bad4b9ebfdb3", + expectedStore: "stored with gitoid", + expectedSearch: "", + expectedError: "value is less than the required length", + }, + { + name: "inexistent payload file", + attestation: "../../../test/missing.attestation.json", + expectedError: "no such file or directory", + }, + } + for _, test := range testTable { + // test `archivistactl store` + e2e.T().Log("Test `archivistactl store` " + test.name) + storeOutput := bytes.NewBufferString("") + rootCmd.SetOut(storeOutput) + rootCmd.SetErr(storeOutput) + rootCmd.SetArgs([]string{"store", test.attestation}) + err := rootCmd.Execute() + if err != nil { + // if return error assert if is expected error from test case + e2e.ErrorContains(err, test.expectedError) + } else { // assert the expected responses + storeActual := storeOutput.String() + e2e.Contains(storeActual, test.expectedStore) + test.gitoidStore = strings.Split(storeActual, "stored with gitoid ")[1] + test.gitoidStore = strings.TrimSuffix(test.gitoidStore, "\n") + } + + // test `archivistactl search` + e2e.T().Log("Test `archivistactl search`" + test.name) + searchOutput := bytes.NewBufferString("") + rootCmd.SetOut(searchOutput) + rootCmd.SetErr(searchOutput) + rootCmd.SetArgs([]string{"search", "sha256:" + test.sha256}) + err = rootCmd.Execute() + if err != nil { + e2e.FailNow(err.Error()) + } + searchActual := searchOutput.String() + e2e.Contains(searchActual, test.expectedSearch) + + if test.expectedRetrieveSub != "" { + // test `archivistactl retrieve subjects` + e2e.T().Log("Test `archivistactl retrieve subjects` " + test.name) + subjectsOutput := bytes.NewBufferString("") + rootCmd.SetOut(subjectsOutput) + rootCmd.SetErr(subjectsOutput) + rootCmd.SetArgs([]string{"retrieve", "subjects", test.gitoidStore}) + err = rootCmd.Execute() + if err != nil { + e2e.FailNow(err.Error()) + } + subjectsActual := subjectsOutput.String() + e2e.Contains(subjectsActual, test.expectedRetrieveSub) + if test.name == "fail attestation" { + e2e.NotContains(subjectsActual, "sha256:"+test.sha256) + } else { + e2e.Contains(subjectsActual, "sha256:"+test.sha256) + } + } + if test.expectedError == "" { + tempDir := os.TempDir() + // test `archivistactl retrieve envelope` + e2e.T().Log("Test `archivistactl retrieve envelope` " + test.name) + envelopeOutput := bytes.NewBufferString("") + rootCmd.SetOut(envelopeOutput) + rootCmd.SetErr(envelopeOutput) + rootCmd.SetArgs([]string{"retrieve", "envelope", test.gitoidStore, "-o", path.Join(tempDir, test.gitoidStore)}) + err = rootCmd.Execute() + if err != nil { + e2e.FailNow(err.Error()) + } + // compares file attestation with the retrieved attestation + fileAtt, err := os.ReadFile(test.attestation) + if err != nil { + e2e.FailNow(err.Error()) + } + fileSaved, err := os.ReadFile(path.Join(tempDir, test.gitoidStore)) + if err != nil { + e2e.FailNow(err.Error()) + } + if err != nil { + e2e.FailNow(err.Error()) + } + e2e.True(bytes.Equal(fileAtt, fileSaved)) + } + } + } + +} diff --git a/cmd/archivistactl/cmd/retrieve.go b/cmd/archivistactl/cmd/retrieve.go index 38d98ae7..3c52ec7b 100644 --- a/cmd/archivistactl/cmd/retrieve.go +++ b/cmd/archivistactl/cmd/retrieve.go @@ -85,7 +85,7 @@ func printSubjects(results retrieveSubjectResults) { digestStrings = append(digestStrings, fmt.Sprintf("%s:%s", digest.Algorithm, digest.Value)) } - fmt.Printf("Name: %s\nDigests: %s\n", edge.Node.Name, strings.Join(digestStrings, ", ")) + rootCmd.Printf("Name: %s\nDigests: %s\n", edge.Node.Name, strings.Join(digestStrings, ", ")) } } diff --git a/cmd/archivistactl/cmd/retrieve_test.go b/cmd/archivistactl/cmd/retrieve_test.go new file mode 100644 index 00000000..2642ff1b --- /dev/null +++ b/cmd/archivistactl/cmd/retrieve_test.go @@ -0,0 +1,94 @@ +// Copyright 2023 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/suite" +) + +// Test Suite: UT Retrieve +type UTRetrieveSuite struct { + suite.Suite +} + +func TestUTRetrieveSuite(t *testing.T) { + suite.Run(t, new(UTRetrieveSuite)) +} + +func (ut *UTRetrieveSuite) Test_MissingSubCommand() { + output := bytes.NewBufferString("") + rootCmd.SetOut(output) + rootCmd.SetErr(output) + rootCmd.SetArgs([]string{"retrieve"}) + err := rootCmd.Execute() + if err != nil { + ut.FailNow("Expected: error") + } + ut.Contains(output.String(), "archivistactl retrieve") +} + +func (ut *UTRetrieveSuite) Test_RetrieveEnvelopeMissingArg() { + output := bytes.NewBufferString("") + rootCmd.SetOut(output) + rootCmd.SetErr(output) + rootCmd.SetArgs([]string{"retrieve", "envelope"}) + err := rootCmd.Execute() + if err != nil { + ut.ErrorContains(err, "accepts 1 arg(s), received 0") + } else { + ut.FailNow("Expected: error") + } +} + +func (ut *UTRetrieveSuite) Test_RetrieveEnvelope() { + output := bytes.NewBufferString("") + rootCmd.SetOut(output) + rootCmd.SetErr(output) + rootCmd.SetArgs([]string{"retrieve", "envelope", "test"}) + err := rootCmd.Execute() + if err != nil { + ut.FailNow("Expected: error") + } + ut.Equal(output.String(), "") +} + +func (ut *UTRetrieveSuite) Test_RetrieveSubjectsMissingArg() { + output := bytes.NewBufferString("") + rootCmd.SetOut(output) + rootCmd.SetErr(output) + rootCmd.SetArgs([]string{"retrieve", "subjects"}) + err := rootCmd.Execute() + if err != nil { + ut.ErrorContains(err, "accepts 1 arg(s), received 0") + } else { + ut.FailNow("Expected: error") + } +} + +func (ut *UTRetrieveSuite) Test_RetrieveSubjectsNoDB() { + output := bytes.NewBufferString("") + rootCmd.SetOut(output) + rootCmd.SetErr(output) + rootCmd.SetArgs([]string{"retrieve", "subjects", "test"}) + err := rootCmd.Execute() + if err != nil { + ut.ErrorContains(err, "connection refused") + } else { + ut.FailNow("Expected: error") + } +} diff --git a/cmd/archivistactl/cmd/root_test.go b/cmd/archivistactl/cmd/root_test.go new file mode 100644 index 00000000..027296cf --- /dev/null +++ b/cmd/archivistactl/cmd/root_test.go @@ -0,0 +1,45 @@ +// Copyright 2023 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/suite" +) + +// Test Suite: UT Root +type UTRootSuite struct { + suite.Suite +} + +func TestUTRootSuite(t *testing.T) { + suite.Run(t, new(UTRootSuite)) +} + +func (ut *UTRootSuite) Test_Root() { + output := bytes.NewBufferString("") + rootCmd.SetOut(output) + rootCmd.SetErr(output) + rootCmd.SetArgs([]string{"help"}) + err := rootCmd.Execute() + if err != nil { + ut.FailNow(err.Error()) + } + actual := output.String() + ut.Contains(actual, "A utility to interact with an archivista server") + +} diff --git a/cmd/archivistactl/cmd/search.go b/cmd/archivistactl/cmd/search.go index bf4a90b6..e41306fd 100644 --- a/cmd/archivistactl/cmd/search.go +++ b/cmd/archivistactl/cmd/search.go @@ -16,7 +16,6 @@ package cmd import ( "errors" - "fmt" "strings" "github.com/in-toto/archivista/pkg/api" @@ -75,14 +74,14 @@ func validateDigestString(ds string) (algo, digest string, err error) { func printResults(results searchResults) { for _, edge := range results.Dsses.Edges { - fmt.Printf("Gitoid: %s\n", edge.Node.GitoidSha256) - fmt.Printf("Collection name: %s\n", edge.Node.Statement.AttestationCollection.Name) + rootCmd.Printf("Gitoid: %s\n", edge.Node.GitoidSha256) + rootCmd.Printf("Collection name: %s\n", edge.Node.Statement.AttestationCollection.Name) types := make([]string, 0, len(edge.Node.Statement.AttestationCollection.Attestations)) for _, attestation := range edge.Node.Statement.AttestationCollection.Attestations { types = append(types, attestation.Type) } - fmt.Printf("Attestations: %s\n\n", strings.Join(types, ", ")) + rootCmd.Printf("Attestations: %s\n\n", strings.Join(types, ", ")) } } diff --git a/cmd/archivistactl/cmd/search_test.go b/cmd/archivistactl/cmd/search_test.go new file mode 100644 index 00000000..02fd9f8c --- /dev/null +++ b/cmd/archivistactl/cmd/search_test.go @@ -0,0 +1,70 @@ +// Copyright 2023 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/suite" +) + +// Test Suite: UT Search +type UTSearchSuite struct { + suite.Suite +} + +func TestUTSearchSuite(t *testing.T) { + suite.Run(t, new(UTSearchSuite)) +} + +func (ut *UTSearchSuite) Test_SearchMissingArgs() { + output := bytes.NewBufferString("") + rootCmd.SetOut(output) + rootCmd.SetErr(output) + rootCmd.SetArgs([]string{"search"}) + err := rootCmd.Execute() + if err != nil { + ut.ErrorContains(err, "expected exactly 1 argument") + } else { + ut.FailNow("Expected: error") + } +} + +func (ut *UTSearchSuite) Test_SearchInvalidDigestString() { + output := bytes.NewBufferString("") + rootCmd.SetOut(output) + rootCmd.SetErr(output) + rootCmd.SetArgs([]string{"search", "invalidDigest"}) + err := rootCmd.Execute() + if err != nil { + ut.ErrorContains(err, "invalid digest string. expected algorithm:digest") + } else { + ut.FailNow("Expected: error") + } +} + +func (ut *UTSearchSuite) Test_NoDB() { + output := bytes.NewBufferString("") + rootCmd.SetOut(output) + rootCmd.SetErr(output) + rootCmd.SetArgs([]string{"search", "sha256:test"}) + err := rootCmd.Execute() + if err != nil { + ut.ErrorContains(err, "connection refused") + } else { + ut.FailNow("Expected: error") + } +} diff --git a/cmd/archivistactl/cmd/store.go b/cmd/archivistactl/cmd/store.go index 8b218dcc..ef7e76a2 100644 --- a/cmd/archivistactl/cmd/store.go +++ b/cmd/archivistactl/cmd/store.go @@ -34,7 +34,7 @@ var ( if gitoid, err := storeAttestationByPath(cmd.Context(), archivistaUrl, filePath); err != nil { return fmt.Errorf("failed to store %s: %w", filePath, err) } else { - fmt.Printf("%s stored with gitoid %s\n", filePath, gitoid) + rootCmd.Printf("%s stored with gitoid %s\n", filePath, gitoid) } } diff --git a/test/test.sh b/test/deploy-services.sh similarity index 70% rename from test/test.sh rename to test/deploy-services.sh index 888ff13f..c9631a43 100755 --- a/test/test.sh +++ b/test/deploy-services.sh @@ -30,10 +30,6 @@ checkprograms() { return $result } -runtests() { - go run "$DIR"/../cmd/archivistactl/main.go store "$DIR"/*.attestation.json -} - waitForArchivista() { echo "Waiting for archivista to be ready..." for attempt in $(seq 1 6); do @@ -55,14 +51,40 @@ if ! checkprograms docker jq ; then exit 1 fi -echo "Test mysql..." -docker compose -f "$DIR/../compose.yml" up --build -d -waitForArchivista -runtests -docker compose -f "$DIR/../compose.yml" down -v - -echo "Test psql..." -docker compose -f "$DIR/../compose.yml" -f "$DIR/../compose-psql.yml" up --build -d -waitForArchivista -runtests -docker compose -f "$DIR/../compose.yml" -f "$DIR/../compose-psql.yml" down -v +startMySQL() { + echo "Test mysql..." + docker compose -f "$DIR/../compose.yml" up --build -d + waitForArchivista +} + +startPgSQL(){ + echo "Test psql..." + docker compose -f "$DIR/../compose.yml" -f "$DIR/../compose-psql.yml" up --build -d + waitForArchivista +} + + +stopServices() { + docker compose -f "$DIR/../compose.yml" -f "$DIR/../compose-psql.yml" down -v +} + + +case $1 in + start-mysql) + startMySQL + waitForArchivista + ;; + + start-pgsql) + startPgSQL + waitForArchivista + ;; + + stop) + stopServices + ;; + + *) + echo "Wrong option. Use $0 start-mysql|start-pgsql|stop" + ;; +esac diff --git a/test/invalid_payload.attestation.json b/test/invalid_payload.attestation.json new file mode 100644 index 00000000..21739a57 --- /dev/null +++ b/test/invalid_payload.attestation.json @@ -0,0 +1,5 @@ +{"payload": "e30K", +"payloadType": "application/vnd.in-toto+json", +"signatures": [{"keyid": "ae2dcc989ea9c109a36e8eba5c4bc16d8fafcfe8e1a614164670d50aedacd647", +"sig": "ahsjnNBEVNqo5/umfwQzWiVAvLx4yk3z6Xh+fsaWyxhmGD1syhOhMOkXapmVvEuscm6la9a/edQKNXXg02ghCw==" +}]} From 9c143ea322e777fc6b7a50d367fcb4b938bbff41 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Sun, 17 Dec 2023 20:26:33 +0100 Subject: [PATCH 2/2] fixup! test: refactoring e2e & adding tests archivistactl Signed-off-by: Kairo de Araujo --- cmd/archivistactl/cmd/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/archivistactl/cmd/e2e_test.go b/cmd/archivistactl/cmd/e2e_test.go index 787c1406..d54320cd 100644 --- a/cmd/archivistactl/cmd/e2e_test.go +++ b/cmd/archivistactl/cmd/e2e_test.go @@ -125,7 +125,7 @@ func (e2e *E2EStoreSuite) Test_E2E() { expectedError: "value is less than the required length", }, { - name: "inexistent payload file", + name: "nonexistent payload file", attestation: "../../../test/missing.attestation.json", expectedError: "no such file or directory", },