From bada5108b60e02d1e56067db9805913d6c9d49a2 Mon Sep 17 00:00:00 2001 From: Mikhail Swift Date: Wed, 7 Feb 2024 15:33:38 -0500 Subject: [PATCH] feat: add ability to distribute artifacts through archivista (#188) * feat: add ability to distribute artifacts through archivista Support use case of private archivista deployments where getting witness may be difficult. Also support use case of folks having custom builds of witness to distribute. Extended to be a more generic artifact store for additional tools. --------- Signed-off-by: Mikhail Swift --- README.md | 4 +- cmd/archivista/main.go | 20 +- docs/docs.go | 221 ++++++++++++++++ docs/swagger.json | 223 +++++++++++++++- docs/swagger.yaml | 146 ++++++++++ internal/artifactstore/artifactstore.go | 205 ++++++++++++++ internal/artifactstore/artifactstore_test.go | 88 ++++++ internal/config/config.go | 3 + internal/server/server.go | 265 ++++++++++++++++++- internal/server/server_test.go | 118 ++++++++- 10 files changed, 1276 insertions(+), 17 deletions(-) create mode 100644 internal/artifactstore/artifactstore.go create mode 100644 internal/artifactstore/artifactstore_test.go diff --git a/README.md b/README.md index 012711a7..c934c57b 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Archivista is configured through environment variables currently. | ARCHIVISTA_SQL_STORE_CONNECTION_STRING | root:example@tcp(db)/testify | SQL store connection string | | ARCHIVISTA_STORAGE_BACKEND | | Backend to use for attestation storage. Options are FILE, BLOB, or empty string for disabled. | | ARCHIVISTA_FILE_SERVE_ON | | What address to serve files on. Only valid when using FILE storage backend. | -| ARCHIVISTA_FILE_DIR | /tmp/archivista/ | Directory to store and serve files. Only valid when using FILE storage backend. | +| ARCHIVISTA_FILE_DIR | /tmp/archivista/ | Directory to store and serve files. Only valid when using FILE storage backend. | | ARCHIVISTA_BLOB_STORE_ENDPOINT | 127.0.0.1:9000 | URL endpoint for blob storage. Only valid when using BLOB storage backend. | | ARCHIVISTA_BLOB_STORE_CREDENTIAL_TYPE | | Blob store credential type. Options are IAM or ACCESS_KEY. | | ARCHIVISTA_BLOB_STORE_ACCESS_KEY_ID | | Blob store access key id. Only valid when using BLOB storage backend. | @@ -89,6 +89,8 @@ Archivista is configured through environment variables currently. | ARCHIVISTA_BLOB_STORE_BUCKET_NAME | | Bucket to use for storage. Only valid when using BLOB storage backend. | | ARCHIVISTA_ENABLE_GRAPHQL | TRUE | Enable GraphQL Endpoint | | ARCHIVISTA_GRAPHQL_WEB_CLIENT_ENABLE | TRUE | Enable GraphiQL, the GraphQL web client | +| ARCHIVISTA_ENABLE_ARTIFACT_STORE | FALSE | Enable Artifact Store Endpoints | +| ARCHIVISTA_ARTIFACT_STORE_CONFIG | /tmp/artifacts/config.yaml | Location of the config describing available artifacts | ## Using Archivista diff --git a/cmd/archivista/main.go b/cmd/archivista/main.go index 7ab1c7a1..0f5c4c82 100644 --- a/cmd/archivista/main.go +++ b/cmd/archivista/main.go @@ -32,6 +32,7 @@ import ( nested "github.com/antonfisher/nested-logrus-formatter" "github.com/gorilla/handlers" + "github.com/in-toto/archivista/internal/artifactstore" "github.com/in-toto/archivista/internal/config" "github.com/in-toto/archivista/internal/metadatastorage/sqlstore" "github.com/in-toto/archivista/internal/objectstorage/blobstore" @@ -56,6 +57,7 @@ func main() { defer cancel() startTime := time.Now() + serverOpts := make([]server.Option, 0) logrus.Infof("executing phase 1: get config from environment (time since start: %s)", time.Since(startTime)) now := time.Now() @@ -81,6 +83,7 @@ func main() { if err != nil { logrus.Fatalf("error initializing storage clients: %+v", err) } + serverOpts = append(serverOpts, server.WithObjectStore(fileStore)) entClient, err := sqlstore.NewEntClient( cfg.SQLStoreBackend, @@ -96,6 +99,7 @@ func main() { if err != nil { logrus.Fatalf("error initializing mysql client: %+v", err) } + serverOpts = append(serverOpts, server.WithMetadataStore(sqlStore)) logrus.WithField("duration", time.Since(now)).Infof("completed phase 3: initializing storage clients") @@ -104,9 +108,23 @@ func main() { // ******************************************************************************** now = time.Now() + // initialize the artifact store + if cfg.EnableArtifactStore { + wds, err := artifactstore.New(artifactstore.WithConfigFile(cfg.ArtifactStoreConfig)) + if err != nil { + logrus.Fatalf("could not create the artifact store: %+v", err) + } + + serverOpts = append(serverOpts, server.WithArtifactStore(wds)) + } + // initialize the server sqlClient := sqlStore.GetClient() - server := server.New(sqlStore, fileStore, cfg, sqlClient) + serverOpts = append(serverOpts, server.WithEntSqlClient(sqlClient)) + server, err := server.New(cfg, serverOpts...) + if err != nil { + logrus.Fatalf("could not create archivista server: %+v", err) + } listenAddress := cfg.ListenOn listenAddress = strings.ToLower(strings.TrimSpace(listenAddress)) diff --git a/docs/docs.go b/docs/docs.go index 7dbeabbf..fcafe3cc 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -81,6 +81,194 @@ const docTemplate = `{ } } }, + "/v1/artifacts": { + "get": { + "description": "retrieves details about all available Artifacts", + "produces": [ + "application/json" + ], + "tags": [ + "Artifacts" + ], + "summary": "List all Artifacts", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Artifact" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/v1/artifacts/{name}": { + "get": { + "description": "retrieves details about all available versions of a specified artifact", + "produces": [ + "application/json" + ], + "tags": [ + "Artifacts" + ], + "summary": "List Artifact Versions", + "parameters": [ + { + "type": "string", + "description": "artifact name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Version" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/v1/artifacts/{name}/{version}": { + "get": { + "description": "retrieves details about a specified version of an artifact", + "produces": [ + "application/json" + ], + "tags": [ + "Artifacts" + ], + "summary": "Artifact Version Details", + "parameters": [ + { + "type": "string", + "description": "artifact name", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "version of artifact", + "name": "version", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/artifactstore.Version" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "404": { + "description": "Not Found" + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "objecpec" + } + } + } + } + }, + "/v1/download/artifact/{name}/{version}/{distribution}": { + "get": { + "description": "downloads a specified distribution of an artifact", + "produces": [ + "application/octet-stream" + ], + "tags": [ + "Artifacts" + ], + "summary": "Download Artifact", + "parameters": [ + { + "type": "string", + "description": "name of artifact", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "version of artifact to download", + "name": "version", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "distribution of artifact to download", + "name": "distribution", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "404": { + "description": "Not Found" + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, "/v1/download/{gitoid}": { "get": { "description": "download an attestation", @@ -178,6 +366,39 @@ const docTemplate = `{ "archivista.Resolver": { "type": "object" }, + "artifactstore.Artifact": { + "type": "object", + "properties": { + "versions": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Version" + } + } + } + }, + "artifactstore.Distribution": { + "type": "object", + "properties": { + "sha256digest": { + "type": "string" + } + } + }, + "artifactstore.Version": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "distributions": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Distribution" + } + } + } + }, "dsse.Envelope": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 323f0c11..09e7a6bc 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -73,6 +73,194 @@ } } }, + "/v1/artifacts": { + "get": { + "description": "retrieves details about all available Artifacts", + "produces": [ + "application/json" + ], + "tags": [ + "Artifacts" + ], + "summary": "List all Artifacts", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Artifact" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/v1/artifacts/{name}": { + "get": { + "description": "retrieves details about all available versions of a specified artifact", + "produces": [ + "application/json" + ], + "tags": [ + "Artifacts" + ], + "summary": "List Artifact Versions", + "parameters": [ + { + "type": "string", + "description": "artifact name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Version" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/v1/artifacts/{name}/{version}": { + "get": { + "description": "retrieves details about a specified version of an artifact", + "produces": [ + "application/json" + ], + "tags": [ + "Artifacts" + ], + "summary": "Artifact Version Details", + "parameters": [ + { + "type": "string", + "description": "artifact name", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "version of artifact", + "name": "version", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/artifactstore.Version" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "404": { + "description": "Not Found" + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "objecpec" + } + } + } + } + }, + "/v1/download/artifact/{name}/{version}/{distribution}": { + "get": { + "description": "downloads a specified distribution of an artifact", + "produces": [ + "application/octet-stream" + ], + "tags": [ + "Artifacts" + ], + "summary": "Download Artifact", + "parameters": [ + { + "type": "string", + "description": "name of artifact", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "version of artifact to download", + "name": "version", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "distribution of artifact to download", + "name": "distribution", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "404": { + "description": "Not Found" + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, "/v1/download/{gitoid}": { "get": { "description": "download an attestation", @@ -170,6 +358,39 @@ "archivista.Resolver": { "type": "object" }, + "artifactstore.Artifact": { + "type": "object", + "properties": { + "versions": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Version" + } + } + } + }, + "artifactstore.Distribution": { + "type": "object", + "properties": { + "sha256digest": { + "type": "string" + } + } + }, + "artifactstore.Version": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "distributions": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Distribution" + } + } + } + }, "dsse.Envelope": { "type": "object", "properties": { @@ -249,4 +470,4 @@ ] } } -} +} \ No newline at end of file diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 9a77dcbc..39c4d533 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -6,6 +6,27 @@ definitions: type: object archivista.Resolver: type: object + artifactstore.Artifact: + properties: + versions: + additionalProperties: + $ref: '#/definitions/artifactstore.Version' + type: object + type: object + artifactstore.Distribution: + properties: + sha256digest: + type: string + type: object + artifactstore.Version: + properties: + description: + type: string + distributions: + additionalProperties: + $ref: '#/definitions/artifactstore.Distribution' + type: object + type: object dsse.Envelope: properties: payload: @@ -107,6 +128,92 @@ paths: schema: $ref: '#/definitions/api.StoreResponse' summary: Upload + /v1/artifacts: + get: + description: retrieves details about all available Artifacts + produces: + - application/json + responses: + "200": + description: OK + schema: + additionalProperties: + $ref: '#/definitions/artifactstore.Artifact' + type: object + "400": + description: Bad Request + schema: + type: string + "500": + description: Internal Server Error + schema: + type: string + summary: List all Artifacts + tags: + - Artifacts + /v1/artifacts/{name}: + get: + description: retrieves details about all available versions of a specified artifact + parameters: + - description: artifact name + in: path + name: name + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + additionalProperties: + $ref: '#/definitions/artifactstore.Version' + type: object + "400": + description: Bad Request + schema: + type: string + "500": + description: Internal Server Error + schema: + type: string + summary: List Artifact Versions + tags: + - Artifacts + /v1/artifacts/{name}/{version}: + get: + description: retrieves details about a specified version of an artifact + parameters: + - description: artifact name + in: path + name: name + required: true + type: string + - description: version of artifact + in: path + name: version + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/artifactstore.Version' + "400": + description: Bad Request + schema: + type: string + "404": + description: Not Found + "500": + description: Internal Server Error + schema: + type: objecpec + summary: Artifact Version Details + tags: + - Artifacts /v1/download/{gitoid}: get: description: download an attestation @@ -136,6 +243,45 @@ paths: summary: Download tags: - attestation + /v1/download/artifact/{name}/{version}/{distribution}: + get: + description: downloads a specified distribution of an artifact + parameters: + - description: name of artifact + in: path + name: name + required: true + type: string + - description: version of artifact to download + in: path + name: version + required: true + type: string + - description: distribution of artifact to download + in: path + name: distribution + required: true + type: string + produces: + - application/octet-stream + responses: + "200": + description: OK + schema: + type: file + "400": + description: Bad Request + schema: + type: string + "404": + description: Not Found + "500": + description: Internal Server Error + schema: + type: string + summary: Download Artifact + tags: + - Artifacts /v1/query: post: description: GraphQL query diff --git a/internal/artifactstore/artifactstore.go b/internal/artifactstore/artifactstore.go new file mode 100644 index 00000000..ba2d6f0b --- /dev/null +++ b/internal/artifactstore/artifactstore.go @@ -0,0 +1,205 @@ +// Copyright 2024 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package artifactstore + +import ( + "crypto" + "errors" + "fmt" + "os" + "strings" + + "github.com/in-toto/go-witness/cryptoutil" + "gopkg.in/yaml.v3" +) + +// Config represents the available artifacts within the store +type Config struct { + Artifacts map[string]Artifact `json:"artifacts"` +} + +// Artifact represents an artifact and it's available versions in the store +type Artifact struct { + Versions map[string]Version `json:"versions"` +} + +// Version represents a version of an Artifact (ex v0.2.0) with the available Distributions of version +type Version struct { + Distributions map[string]Distribution `json:"distributions"` + Description string `json:"description"` +} + +// Distribution is a specific distribution of a Version of an Artifact(ex linux-amd64) +type Distribution struct { + FileLocation string `json:"-"` + SHA256Digest string `json:"sha256digest"` +} + +// Store is an artifact store served from Archivista +type Store struct { + config Config +} + +type Option func(*Store) error + +// WithConfig creates a Store with the provided config +func WithConfig(config Config) Option { + return func(as *Store) error { + as.config = config + return nil + } +} + +// WithConfigFile creates a Store with a config loaded from a yaml file on disk +func WithConfigFile(configPath string) Option { + return func(as *Store) error { + configBytes, err := os.ReadFile(configPath) + if err != nil { + return err + } + + config := Config{} + if err := yaml.Unmarshal(configBytes, &config); err != nil { + return err + } + + return WithConfig(config)(as) + } +} + +// New creates a new Store with the provided options +func New(opts ...Option) (Store, error) { + as := Store{} + errs := make([]error, 0) + for _, opt := range opts { + if err := opt(&as); err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return Store{}, errors.Join(errs...) + } + + if err := verifyConfig(as); err != nil { + return Store{}, err + } + + return as, nil +} + +// verifyConfig ensures that each file exists on disk and that the sha256sum of the +// files on disk match those of the config +func verifyConfig(as Store) error { + errs := make([]error, 0) + for artifactName, artifact := range as.config.Artifacts { + for versionString, version := range artifact.Versions { + for distroString, distro := range version.Distributions { + if _, err := os.Stat(distro.FileLocation); err != nil { + errs = append(errs, fmt.Errorf("%v version %v-%v does not exist on disk: %w", artifactName, versionString, distroString, err)) + continue + } + + digestSet, err := cryptoutil.CalculateDigestSetFromFile(distro.FileLocation, []crypto.Hash{crypto.SHA256}) + if err != nil { + errs = append(errs, fmt.Errorf("could not calculate sha256 digest for %v version %v-%v: %w", artifactName, versionString, distroString, err)) + } + + sha256Digest := digestSet[cryptoutil.DigestValue{Hash: crypto.SHA256, GitOID: false}] + if !strings.EqualFold(sha256Digest, distro.SHA256Digest) { + errs = append(errs, fmt.Errorf("sha256 digest of %v version %v-%v does not match config: got %v, expected %v", artifactName, versionString, distroString, sha256Digest, distro.SHA256Digest)) + } + } + } + } + + if len(errs) > 0 { + return errors.Join(errs...) + } + + return nil +} + +// Artifacts returns a copy of all the Store's Artifacts +func (as Store) Artifacts() map[string]Artifact { + out := make(map[string]Artifact) + for artifactString, artifact := range as.config.Artifacts { + out[artifactString] = artifact + } + + return out +} + +// Versions returns all of the available Versions for an Artifact. +func (as Store) Versions(artifact string) (map[string]Version, bool) { + out := make(map[string]Version) + a, ok := as.config.Artifacts[artifact] + if !ok { + return out, false + } + + for verString, version := range a.Versions { + out[verString] = version + } + + return out, ok +} + +// Version returns a specific Version for an artifact, if it exists +func (as Store) Version(artifact, version string) (Version, bool) { + a, ok := as.config.Artifacts[artifact] + if !ok { + return Version{}, false + } + + v, ok := a.Versions[version] + return v, ok +} + +// Distributions returns all of the available Distributions for a specified Version of an Artifact +func (as Store) Distributions(artifact, version string) (map[string]Distribution, bool) { + out := make(map[string]Distribution) + a, ok := as.config.Artifacts[artifact] + if !ok { + return out, false + } + + vers, ok := a.Versions[version] + if !ok { + return out, ok + } + + for distroString, distro := range vers.Distributions { + out[distroString] = distro + } + + return out, true +} + +// Distribution returns the entry for a specific distribution for a specific version +func (as Store) Distribution(artifact, version, distribution string) (Distribution, bool) { + a, ok := as.config.Artifacts[artifact] + if !ok { + return Distribution{}, false + } + + vers, ok := a.Versions[version] + if !ok { + return Distribution{}, false + } + + distro, ok := vers.Distributions[distribution] + return distro, ok +} diff --git a/internal/artifactstore/artifactstore_test.go b/internal/artifactstore/artifactstore_test.go new file mode 100644 index 00000000..605ae1e5 --- /dev/null +++ b/internal/artifactstore/artifactstore_test.go @@ -0,0 +1,88 @@ +// Copyright 2024 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package artifactstore + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func createTestConfigFile(t *testing.T, workingDir, version, distroFilePath, distroDigest string) string { + testConfig := `artifacts: + witness: + versions: + ` + version + `: + description: test + distributions: + linux: + filelocation: ` + distroFilePath + ` + sha256digest: ` + distroDigest + testConfigFilePath := filepath.Join(workingDir, "config.yaml") + testConfigFile, err := os.Create(testConfigFilePath) + require.NoError(t, err) + _, err = testConfigFile.WriteString(testConfig) + require.NoError(t, err) + require.NoError(t, testConfigFile.Close()) + return testConfigFilePath +} + +func TestStore(t *testing.T) { + workingDir := t.TempDir() + testDistroFilePath := filepath.Join(workingDir, "test") + testDistroDigest := "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + testVersion := "v0.1.0" + testDistroFile, err := os.Create(testDistroFilePath) + require.NoError(t, err) + _, err = testDistroFile.Write([]byte("test")) + require.NoError(t, err) + require.NoError(t, testDistroFile.Close()) + + t.Run("all good", func(t *testing.T) { + testArtifactName := "witness" + testDistroName := "linux" + testConfigFilePath := createTestConfigFile(t, workingDir, testVersion, testDistroFilePath, testDistroDigest) + as, err := New(WithConfigFile(testConfigFilePath)) + require.NoError(t, err) + artifacts := as.Artifacts() + assert.Len(t, artifacts, 1) + versions, ok := as.Versions(testArtifactName) + assert.True(t, ok) + assert.Len(t, versions, 1) + version, ok := as.Version(testArtifactName, testVersion) + assert.True(t, ok) + assert.Len(t, version.Distributions, 1) + testDistro, ok := as.Distribution(testArtifactName, testVersion, testDistroName) + assert.True(t, ok) + assert.Equal(t, testDistro.FileLocation, testDistroFilePath) + assert.Equal(t, testDistro.SHA256Digest, testDistroDigest) + }) + + t.Run("wrong file path", func(t *testing.T) { + testConfigFilePath := createTestConfigFile(t, workingDir, testVersion, "garbage", testDistroDigest) + _, err := New(WithConfigFile(testConfigFilePath)) + assert.Error(t, err) + + }) + + t.Run("wrong file digest", func(t *testing.T) { + testConfigFilePath := createTestConfigFile(t, workingDir, testVersion, testDistroFilePath, "garbage") + _, err := New(WithConfigFile(testConfigFilePath)) + assert.Error(t, err) + }) +} diff --git a/internal/config/config.go b/internal/config/config.go index bbdc3ae9..dd708fcb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,6 +51,9 @@ type Config struct { EnableGraphql bool `default:"TRUE" desc:"*** Enable GraphQL Endpoint" split_words:"true"` GraphqlWebClientEnable bool `default:"TRUE" desc:"Enable GraphiQL, the GraphQL web client" split_words:"true"` + + EnableArtifactStore bool `default:"FALSE" desc:"*** Enable Artifact Store Endpoints" split_words:"true"` + ArtifactStoreConfig string `default:"/tmp/artifacts/config.yaml" desc:"Location of the config describing available artifacts" split_words:"true"` } // Process reads config from env diff --git a/internal/server/server.go b/internal/server/server.go index 4cc8347f..165639f5 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -22,6 +22,9 @@ import ( "fmt" "io" "net/http" + "os" + "path/filepath" + "strconv" "strings" "entgo.io/contrib/entgql" @@ -32,6 +35,7 @@ import ( "github.com/in-toto/archivista" _ "github.com/in-toto/archivista/docs" "github.com/in-toto/archivista/ent" + "github.com/in-toto/archivista/internal/artifactstore" "github.com/in-toto/archivista/internal/config" "github.com/in-toto/archivista/pkg/api" "github.com/sirupsen/logrus" @@ -41,7 +45,9 @@ import ( type Server struct { metadataStore Storer objectStore StorerGetter + artifactStore artifactstore.Store router *mux.Router + sqlClient *ent.Client } type Storer interface { @@ -57,16 +63,48 @@ type StorerGetter interface { Getter } -func New(metadataStore Storer, objectStore StorerGetter, cfg *config.Config, sqlClient *ent.Client) Server { +type Option func(*Server) + +func WithMetadataStore(metadataStore Storer) Option { + return func(s *Server) { + s.metadataStore = metadataStore + } +} + +func WithObjectStore(objectStore StorerGetter) Option { + return func(s *Server) { + s.objectStore = objectStore + } +} + +func WithEntSqlClient(sqlClient *ent.Client) Option { + return func(s *Server) { + s.sqlClient = sqlClient + } +} + +func WithArtifactStore(wds artifactstore.Store) Option { + return func(s *Server) { + s.artifactStore = wds + } +} + +func New(cfg *config.Config, opts ...Option) (Server, error) { r := mux.NewRouter() - s := &Server{metadataStore, objectStore, nil} + s := Server{ + router: r, + } + + for _, opt := range opts { + opt(&s) + } // TODO: remove from future version (v0.6.0) endpoint with version r.HandleFunc("/download/{gitoid}", s.DownloadHandler) r.HandleFunc("/upload", s.UploadHandler) if cfg.EnableGraphql { - r.Handle("/query", s.Query(sqlClient)) - r.Handle("/v1/query", s.Query(sqlClient)) + r.Handle("/query", s.Query(s.sqlClient)) + r.Handle("/v1/query", s.Query(s.sqlClient)) } r.HandleFunc("/v1/download/{gitoid}", s.DownloadHandler) @@ -76,11 +114,16 @@ func New(metadataStore Storer, objectStore StorerGetter, cfg *config.Config, sql playground.Handler("Archivista", "/v1/query"), ) } - r.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler) - s.router = r - return *s + if cfg.EnableArtifactStore { + r.HandleFunc("/v1/artifacts", s.AllArtifactsHandler) + r.HandleFunc("/v1/artifacts/{name}", s.ArtifactAllVersionsHandler) + r.HandleFunc("/v1/artifacts/{name}/{version}", s.ArtifactVersionHandler) + r.HandleFunc("/v1/download/artifact/{name}/{version}/{distribution}", s.DownloadArtifactHandler) + } + r.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler) + return s, nil } // @title Archivista API @@ -233,3 +276,211 @@ func (s *Server) Query(sqlclient *ent.Client) *handler.Server { srv.Use(entgql.Transactioner{TxOpener: sqlclient}) return srv } + +// @Summary List all Artifacts +// @Description retrieves details about all available Artifacts +// @Produce json +// @Success 200 {object} map[string]artifactstore.Artifact +// @Failure 500 {object} string +// @Failure 400 {object} string +// @Tags Artifacts +// @Router /v1/artifacts [get] +func (s *Server) AllArtifactsHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, fmt.Sprintf("%s is an unsupported method", r.Method), http.StatusBadRequest) + return + } + + allArtifacts := s.artifactStore.Artifacts() + allArtifactsJson, err := json.Marshal(allArtifacts) + if err != nil { + http.Error(w, fmt.Errorf("could not marshal artifact versions: %w", err).Error(), http.StatusInternalServerError) + return + } + + if _, err := io.Copy(w, bytes.NewReader(allArtifactsJson)); err != nil { + http.Error(w, fmt.Errorf("could not send json response: %w", err).Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + +} + +// @Summary List Artifact Versions +// @Description retrieves details about all available versions of a specified artifact +// @Produce json +// @Param name path string true "artifact name" +// @Success 200 {object} map[string]artifactstore.Version +// @Failure 500 {object} string +// @Failure 400 {object} string +// @Tags Artifacts +// @Router /v1/artifacts/{name} [get] +func (s *Server) ArtifactAllVersionsHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, fmt.Sprintf("%s is an unsupported method", r.Method), http.StatusBadRequest) + return + } + + vars := mux.Vars(r) + if vars == nil { + http.Error(w, "name parameter is required", http.StatusBadRequest) + return + } + + artifactName := vars["name"] + if len(artifactName) == 0 { + http.Error(w, "name parameter is required", http.StatusBadRequest) + return + } + + artifactVersions, ok := s.artifactStore.Versions(artifactName) + if !ok { + http.Error(w, "artifact not found", http.StatusNotFound) + return + } + + artifactVersionsJson, err := json.Marshal(artifactVersions) + if err != nil { + http.Error(w, fmt.Errorf("could not marshal artifact versions: %w", err).Error(), http.StatusInternalServerError) + return + } + + if _, err := io.Copy(w, bytes.NewReader(artifactVersionsJson)); err != nil { + http.Error(w, fmt.Errorf("could not send json response: %w", err).Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") +} + +// @Summary Artifact Version Details +// @Description retrieves details about a specified version of an artifact +// @Produce json +// @Param name path string true "artifact name" +// @Param version path string true "version of artifact" +// @Success 200 {object} artifactstore.Version +// @Failure 500 {objecpec} string +// @Failure 404 {object} nil +// @Failure 400 {object} string +// @Tags Artifacts +// @Router /v1/artifacts/{name}/{version} [get] +func (s *Server) ArtifactVersionHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, fmt.Sprintf("%s is an unsupported method", r.Method), http.StatusBadRequest) + return + } + + vars := mux.Vars(r) + if vars == nil { + http.Error(w, "name and version parameters are required", http.StatusBadRequest) + return + } + + artifactString := vars["name"] + if len(artifactString) == 0 { + http.Error(w, "name parameter is required", http.StatusBadRequest) + return + } + + versionString := vars["version"] + if len(versionString) == 0 { + http.Error(w, "version parameter is required", http.StatusBadRequest) + return + } + + version, ok := s.artifactStore.Version(artifactString, versionString) + if !ok { + http.Error(w, "version not found", http.StatusNotFound) + return + } + + versionJson, err := json.Marshal(version) + if err != nil { + http.Error(w, fmt.Errorf("could not marshal artifact distros: %w", err).Error(), http.StatusInternalServerError) + return + } + + if _, err := io.Copy(w, bytes.NewReader(versionJson)); err != nil { + http.Error(w, fmt.Errorf("could not send json response: %w", err).Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") +} + +// @Summary Download Artifact +// @Description downloads a specified distribution of an artifact +// @Produce octet-stream +// @Param name path string true "name of artifact" +// @Param version path string true "version of artifact to download" +// @Param distribution path string true "distribution of artifact to download" +// @Success 200 {file} octet-stream +// @Failure 500 {object} string +// @Failure 404 {object} nil +// @Failure 400 {object} string +// @Tags Artifacts +// @Router /v1/download/artifact/{name}/{version}/{distribution} [get] +func (s *Server) DownloadArtifactHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, fmt.Sprintf("%s is an unsupported method", r.Method), http.StatusBadRequest) + return + } + + vars := mux.Vars(r) + if vars == nil { + http.Error(w, "version and distribution parameter is required", http.StatusBadRequest) + return + } + + artifactString := vars["name"] + if len(artifactString) == 0 { + http.Error(w, "name parameter is required", http.StatusBadRequest) + return + } + + versionString := vars["version"] + if len(versionString) == 0 { + http.Error(w, "version parameter is required", http.StatusBadRequest) + return + } + + distroString := vars["distribution"] + if len(distroString) == 0 { + http.Error(w, "distribution parameter is required", http.StatusBadRequest) + return + } + + distro, ok := s.artifactStore.Distribution(artifactString, versionString, distroString) + if !ok { + http.Error(w, "distribution of artifact not found", http.StatusNotFound) + return + } + + file, err := os.Open(distro.FileLocation) + if err != nil { + http.Error(w, "could not read distribution file", http.StatusBadRequest) + return + } + + defer func() { + if err := file.Close(); err != nil { + logrus.Errorf(fmt.Sprintf("failed to close artifact distribution file %s: %+v", distro.FileLocation, err)) + } + }() + + fileInfo, err := file.Stat() + if err != nil { + http.Error(w, "could not stat distribution file", http.StatusBadRequest) + return + } + + fileName := filepath.Base(distro.FileLocation) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%v", fileName)) + w.Header().Set("Content-Length", strconv.FormatInt(fileInfo.Size(), 10)) + if _, err := io.Copy(w, file); err != nil { + http.Error(w, fmt.Errorf("could not send artifact distribution: %w", err).Error(), http.StatusInternalServerError) + return + } +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 92a40856..420455cb 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -21,10 +21,13 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" "github.com/gorilla/mux" + "github.com/in-toto/archivista/internal/artifactstore" "github.com/in-toto/archivista/internal/config" "github.com/in-toto/archivista/pkg/api" "github.com/stretchr/testify/mock" @@ -103,20 +106,26 @@ func (ut *UTServerSuite) SetupTest() { ut.mockedStorer = new(StorerMock) ut.mockedStorerGetter = new(StorerGetterMock) ut.mockedResposeRecorder = new(ResponseRecorderMock) - ut.testServer = Server{ut.mockedStorer, ut.mockedStorerGetter, nil} + ut.testServer = Server{ + metadataStore: ut.mockedStorer, + objectStore: ut.mockedStorerGetter, + artifactStore: ut.testArtifactStore(), + } } func (ut *UTServerSuite) Test_New() { cfg := new(config.Config) cfg.EnableGraphql = true cfg.GraphqlWebClientEnable = true - ut.testServer = New(ut.mockedStorer, ut.mockedStorerGetter, cfg, nil) + var err error + ut.testServer, err = New(cfg, WithMetadataStore(ut.mockedStorer), WithObjectStore(ut.mockedStorerGetter)) + ut.NoError(err) ut.NotNil(ut.testServer) router := ut.testServer.Router() ut.NotNil(router) allPaths := []string{} - err := router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { + err = router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { pathTemplate, err := route.GetPathTemplate() if err != nil { ut.FailNow(err.Error()) @@ -124,6 +133,7 @@ func (ut *UTServerSuite) Test_New() { allPaths = append(allPaths, pathTemplate) return nil }) + if err != nil { ut.FailNow(err.Error()) } @@ -141,13 +151,15 @@ func (ut *UTServerSuite) Test_New_EnableGraphQL_False() { cfg := new(config.Config) cfg.EnableGraphql = false cfg.GraphqlWebClientEnable = true - ut.testServer = New(ut.mockedStorer, ut.mockedStorerGetter, cfg, nil) + var err error + ut.testServer, err = New(cfg, WithMetadataStore(ut.mockedStorer), WithObjectStore(ut.mockedStorerGetter)) + ut.NoError(err) ut.NotNil(ut.testServer) router := ut.testServer.Router() ut.NotNil(router) allPaths := []string{} - err := router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { + err = router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { pathTemplate, err := route.GetPathTemplate() if err != nil { ut.FailNow(err.Error()) @@ -173,13 +185,15 @@ func (ut *UTServerSuite) Test_New_GraphqlWebClientEnable_False() { cfg := new(config.Config) cfg.EnableGraphql = true cfg.GraphqlWebClientEnable = false - ut.testServer = New(ut.mockedStorer, ut.mockedStorerGetter, cfg, nil) + var err error + ut.testServer, err = New(cfg, WithMetadataStore(ut.mockedStorer), WithObjectStore(ut.mockedStorerGetter)) + ut.NoError(err) ut.NotNil(ut.testServer) router := ut.testServer.Router() ut.NotNil(router) allPaths := []string{} - err := router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { + err = router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { pathTemplate, err := route.GetPathTemplate() if err != nil { ut.FailNow(err.Error()) @@ -386,3 +400,93 @@ func (ut *UTServerSuite) Test_DownloadHandler_NotFound() { ut.Equal(http.StatusNotFound, ut.mockedResposeRecorder.Code) ut.Nil(ut.mockedResposeRecorder.Body) } + +func (ut *UTServerSuite) Test_AllArtifactsHandler() { + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/artifacts", nil) + + ut.testServer.AllArtifactsHandler(w, request) + ut.Equal(http.StatusOK, w.Code) + ut.Contains(w.Body.String(), "witness") + ut.Contains(w.Body.String(), "v0.1.0") + ut.Contains(w.Body.String(), "linux-x64") +} + +func (ut *UTServerSuite) Test_ArtifactAllVersionsHandler() { + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/artifacts/witness", nil) + request = mux.SetURLVars(request, map[string]string{"name": "witness"}) + + ut.testServer.ArtifactAllVersionsHandler(w, request) + ut.Equal(http.StatusOK, w.Code) + ut.Contains(w.Body.String(), "v0.1.0") + ut.Contains(w.Body.String(), "linux-x64") +} + +func (ut *UTServerSuite) Test_ArtifactVersionHandler() { + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/artifacts/witness/v0.1.0", nil) + request = mux.SetURLVars(request, map[string]string{"name": "witness", "version": "v0.1.0"}) + + ut.testServer.ArtifactVersionHandler(w, request) + ut.Equal(http.StatusOK, w.Code) + ut.Contains(w.Body.String(), "linux-x64") +} + +func (ut *UTServerSuite) Test_ArtifactVersionHandler_NotFound() { + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/artifacts/witness/v0.3.0", nil) + request = mux.SetURLVars(request, map[string]string{"name": "witness", "version": "v0.3.0"}) + + ut.testServer.ArtifactVersionHandler(w, request) + ut.Equal(http.StatusNotFound, w.Code) + ut.Contains(w.Body.String(), "version not found") +} + +func (ut *UTServerSuite) Test_DownloadArtifactHandler() { + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/download/artifact/witness/v0.1.0/linux-x64", nil) + request = mux.SetURLVars(request, map[string]string{"name": "witness", "version": "v0.1.0", "distribution": "linux-x64"}) + + ut.testServer.DownloadArtifactHandler(w, request) + ut.Equal(http.StatusOK, w.Code) + ut.Contains(w.Body.String(), "test") +} + +func (ut *UTServerSuite) Test_DownloadArtifactHandler_NotFound() { + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/download/artifact/witness/v0.1.0/linux-arm", nil) + request = mux.SetURLVars(request, map[string]string{"name": "witness", "version": "v0.1.0", "distribution": "linux-arm"}) + + ut.testServer.DownloadArtifactHandler(w, request) + ut.Equal(http.StatusNotFound, w.Code) + ut.Contains(w.Body.String(), "distribution of artifact not found") +} + +func (ut *UTServerSuite) testArtifactStore() artifactstore.Store { + testDir := ut.T().TempDir() + testDistroFilePath := filepath.Join(testDir, "witness-v0.1.0-linux-x64") + ut.NoError(os.WriteFile(testDistroFilePath, []byte("test"), 0644)) + + config := artifactstore.Config{ + Artifacts: map[string]artifactstore.Artifact{ + "witness": { + Versions: map[string]artifactstore.Version{ + "v0.1.0": { + Description: "some description", + Distributions: map[string]artifactstore.Distribution{ + "linux-x64": { + SHA256Digest: "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + FileLocation: testDistroFilePath, + }, + }, + }, + }, + }, + }, + } + + wds, err := artifactstore.New(artifactstore.WithConfig(config)) + ut.NoError(err) + return wds +}