From 2eab1165895f7871a6ab706354fd62a13cfe88a7 Mon Sep 17 00:00:00 2001 From: Kairo Araujo Date: Tue, 4 Jun 2024 05:12:38 +0200 Subject: [PATCH] Refactoring services (#287) * refactor: remove iternal for Archivista flexibility Remove the internal in order to enable Archivista source code user to: - Extend/Modify the existent services (config, storage, etc) within their code base - Allow to use Archivista encapuslated in other services, example API service Signed-off-by: Kairo Araujo * refactor: simplify archivista main cmd Simplifies Archivista main cmd moving all the service logic into the `pkg/server`. It allows users to use the Archivista Server instance as an Service Interface allowing to integrate along to API services. Signed-off-by: Kairo Araujo --------- Signed-off-by: Kairo Araujo --- cmd/archivista/main.go | 123 ++--------- .../artifactstore/artifactstore.go | 0 .../artifactstore/artifactstore_test.go | 0 {internal => pkg}/config/config.go | 0 {internal => pkg}/config/config_test.go | 0 .../attestationcollection/parserstorer.go | 2 +- {internal => pkg}/metadatastorage/parser.go | 0 .../parserregistry/registry.go | 4 +- .../metadatastorage/sqlstore/client.go | 0 .../metadatastorage/sqlstore/store.go | 4 +- .../objectstorage/blobstore/minio.go | 0 .../objectstorage/filestore/file.go | 0 .../objectstorage/filestore/file_test.go | 2 +- {internal => pkg}/server/server.go | 4 +- {internal => pkg}/server/server_test.go | 4 +- pkg/server/services.go | 191 ++++++++++++++++++ 16 files changed, 213 insertions(+), 121 deletions(-) rename {internal => pkg}/artifactstore/artifactstore.go (100%) rename {internal => pkg}/artifactstore/artifactstore_test.go (100%) rename {internal => pkg}/config/config.go (100%) rename {internal => pkg}/config/config_test.go (100%) rename {internal => pkg}/metadatastorage/attestationcollection/parserstorer.go (97%) rename {internal => pkg}/metadatastorage/parser.go (100%) rename {internal => pkg}/metadatastorage/parserregistry/registry.go (86%) rename {internal => pkg}/metadatastorage/sqlstore/client.go (100%) rename {internal => pkg}/metadatastorage/sqlstore/store.go (98%) rename {internal => pkg}/objectstorage/blobstore/minio.go (100%) rename {internal => pkg}/objectstorage/filestore/file.go (100%) rename {internal => pkg}/objectstorage/filestore/file_test.go (97%) rename {internal => pkg}/server/server.go (99%) rename {internal => pkg}/server/server_test.go (99%) create mode 100644 pkg/server/services.go diff --git a/cmd/archivista/main.go b/cmd/archivista/main.go index 6d8606a4..7a5448a6 100644 --- a/cmd/archivista/main.go +++ b/cmd/archivista/main.go @@ -21,7 +21,6 @@ package main import ( "context" - "fmt" "net" "net/http" "os" @@ -32,13 +31,7 @@ import ( nested "github.com/antonfisher/nested-logrus-formatter" "github.com/gorilla/handlers" - "github.com/in-toto/archivista/internal/artifactstore" - "github.com/in-toto/archivista/internal/config" - "github.com/in-toto/archivista/internal/metadatastorage/sqlstore" - "github.com/in-toto/archivista/internal/objectstorage/blobstore" - "github.com/in-toto/archivista/internal/objectstorage/filestore" - "github.com/in-toto/archivista/internal/server" - "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/in-toto/archivista/pkg/server" "github.com/sirupsen/logrus" ) @@ -57,76 +50,19 @@ func main() { defer cancel() startTime := time.Now() - serverOpts := make([]server.Option, 0) - logrus.Infof("executing phase 1: get config from environment (time since start: %s)", time.Since(startTime)) - now := time.Now() - - cfg := new(config.Config) - if err := cfg.Process(); err != nil { - logrus.Fatal(err) - } - - level, err := logrus.ParseLevel(cfg.LogLevel) - if err != nil { - logrus.Fatalf("invalid log level %s", cfg.LogLevel) - } - logrus.SetLevel(level) - - logrus.WithField("duration", time.Since(now)).Infof("completed phase 1: get config from environment") - - // ******************************************************************************** - logrus.Infof("executing phase 2: initializing storage clients (time since start: %s)", time.Since(startTime)) - // ******************************************************************************** - now = time.Now() - fileStore, fileStoreCh, err := initObjectStore(ctx, cfg) - if err != nil { - logrus.Fatalf("error initializing storage clients: %+v", err) - } - serverOpts = append(serverOpts, server.WithObjectStore(fileStore)) + archivistaService := &server.ArchivistaService{Ctx: ctx, Cfg: nil} - entClient, err := sqlstore.NewEntClient( - cfg.SQLStoreBackend, - cfg.SQLStoreConnectionString, - sqlstore.ClientWithMaxIdleConns(cfg.SQLStoreMaxIdleConnections), - sqlstore.ClientWithMaxOpenConns(cfg.SQLStoreMaxOpenConnections), - sqlstore.ClientWithConnMaxLifetime(cfg.SQLStoreConnectionMaxLifetime)) + server, err := archivistaService.Setup() if err != nil { - logrus.Fatalf("could not create ent client: %+v", err) + logrus.Fatalf("unable to setup archivista service: %+v", err) } - - sqlStore, sqlStoreCh, err := sqlstore.New(ctx, entClient) - if err != nil { - logrus.Fatalf("error initializing mysql client: %+v", err) - } - serverOpts = append(serverOpts, server.WithMetadataStore(sqlStore)) - - logrus.WithField("duration", time.Since(now)).Infof("completed phase 3: initializing storage clients") - // ******************************************************************************** - logrus.Infof("executing phase 3: create and register http service (time since start: %s)", time.Since(startTime)) + logrus.Infof("executing phase: create and register http service (time since start: %s)", time.Since(startTime)) // ******************************************************************************** - now = time.Now() - - // initialize the artifact store - if cfg.EnableArtifactStore { - wds, err := artifactstore.New(artifactstore.WithConfigFile(cfg.ArtifactStoreConfig)) - if err != nil { - logrus.Fatalf("could not create the artifact store: %+v", err) - } - - serverOpts = append(serverOpts, server.WithArtifactStore(wds)) - } - - // initialize the server - sqlClient := sqlStore.GetClient() - serverOpts = append(serverOpts, server.WithEntSqlClient(sqlClient)) - server, err := server.New(cfg, serverOpts...) - if err != nil { - logrus.Fatalf("could not create archivista server: %+v", err) - } + now := time.Now() - listenAddress := cfg.ListenOn + listenAddress := archivistaService.Cfg.ListenOn listenAddress = strings.ToLower(strings.TrimSpace(listenAddress)) proto := "" if strings.HasPrefix(listenAddress, "tcp://") { @@ -143,10 +79,10 @@ func main() { } srv := &http.Server{ Handler: handlers.CORS( - handlers.AllowedOrigins(cfg.CORSAllowOrigins), + handlers.AllowedOrigins(archivistaService.Cfg.CORSAllowOrigins), handlers.AllowedMethods([]string{"GET", "POST", "OPTIONS"}), handlers.AllowedHeaders([]string{"Accept", "Content-Type", "Content-Length", "Accept-Encoding", "X-CSRF-Token", "Authorization"}), - )(server.Router()), + )(server.Router()), ReadTimeout: 5 * time.Second, WriteTimeout: 10 * time.Second, } @@ -156,47 +92,12 @@ func main() { } }() - logrus.WithField("duration", time.Since(now)).Infof("completed phase 5: create and register http service") + logrus.WithField("duration", time.Since(now)).Infof("completed phase: create and register http service") logrus.Infof("startup complete (time since start: %s)", time.Since(startTime)) <-ctx.Done() - <-fileStoreCh - <-sqlStoreCh + <-archivistaService.GetFileStoreCh() + <-archivistaService.GetSQLStoreCh() logrus.Infof("exiting, uptime: %v", time.Since(startTime)) } - -func initObjectStore(ctx context.Context, cfg *config.Config) (server.StorerGetter, <-chan error, error) { - switch strings.ToUpper(cfg.StorageBackend) { - case "FILE": - return filestore.New(ctx, cfg.FileDir, cfg.FileServeOn) - - case "BLOB": - var creds *credentials.Credentials - if cfg.BlobStoreCredentialType == "IAM" { - creds = credentials.NewIAM("") - } else if cfg.BlobStoreCredentialType == "ACCESS_KEY" { - creds = credentials.NewStaticV4(cfg.BlobStoreAccessKeyId, cfg.BlobStoreSecretAccessKeyId, "") - } else { - logrus.Fatalln("invalid blob store credential type: ", cfg.BlobStoreCredentialType) - } - return blobstore.New( - ctx, - cfg.BlobStoreEndpoint, - creds, - cfg.BlobStoreBucketName, - cfg.BlobStoreUseTLS, - ) - - case "": - errCh := make(chan error) - go func() { - <-ctx.Done() - close(errCh) - }() - return nil, errCh, nil - - default: - return nil, nil, fmt.Errorf("unknown storage backend: %s", cfg.StorageBackend) - } -} diff --git a/internal/artifactstore/artifactstore.go b/pkg/artifactstore/artifactstore.go similarity index 100% rename from internal/artifactstore/artifactstore.go rename to pkg/artifactstore/artifactstore.go diff --git a/internal/artifactstore/artifactstore_test.go b/pkg/artifactstore/artifactstore_test.go similarity index 100% rename from internal/artifactstore/artifactstore_test.go rename to pkg/artifactstore/artifactstore_test.go diff --git a/internal/config/config.go b/pkg/config/config.go similarity index 100% rename from internal/config/config.go rename to pkg/config/config.go diff --git a/internal/config/config_test.go b/pkg/config/config_test.go similarity index 100% rename from internal/config/config_test.go rename to pkg/config/config_test.go diff --git a/internal/metadatastorage/attestationcollection/parserstorer.go b/pkg/metadatastorage/attestationcollection/parserstorer.go similarity index 97% rename from internal/metadatastorage/attestationcollection/parserstorer.go rename to pkg/metadatastorage/attestationcollection/parserstorer.go index 3c03b41d..e456940e 100644 --- a/internal/metadatastorage/attestationcollection/parserstorer.go +++ b/pkg/metadatastorage/attestationcollection/parserstorer.go @@ -20,7 +20,7 @@ import ( "github.com/google/uuid" "github.com/in-toto/archivista/ent" - "github.com/in-toto/archivista/internal/metadatastorage" + "github.com/in-toto/archivista/pkg/metadatastorage" "github.com/in-toto/go-witness/attestation" ) diff --git a/internal/metadatastorage/parser.go b/pkg/metadatastorage/parser.go similarity index 100% rename from internal/metadatastorage/parser.go rename to pkg/metadatastorage/parser.go diff --git a/internal/metadatastorage/parserregistry/registry.go b/pkg/metadatastorage/parserregistry/registry.go similarity index 86% rename from internal/metadatastorage/parserregistry/registry.go rename to pkg/metadatastorage/parserregistry/registry.go index 08ef91e4..eff278d4 100644 --- a/internal/metadatastorage/parserregistry/registry.go +++ b/pkg/metadatastorage/parserregistry/registry.go @@ -15,8 +15,8 @@ package parserregistry import ( - "github.com/in-toto/archivista/internal/metadatastorage" - "github.com/in-toto/archivista/internal/metadatastorage/attestationcollection" + "github.com/in-toto/archivista/pkg/metadatastorage" + "github.com/in-toto/archivista/pkg/metadatastorage/attestationcollection" ) var ( diff --git a/internal/metadatastorage/sqlstore/client.go b/pkg/metadatastorage/sqlstore/client.go similarity index 100% rename from internal/metadatastorage/sqlstore/client.go rename to pkg/metadatastorage/sqlstore/client.go diff --git a/internal/metadatastorage/sqlstore/store.go b/pkg/metadatastorage/sqlstore/store.go similarity index 98% rename from internal/metadatastorage/sqlstore/store.go rename to pkg/metadatastorage/sqlstore/store.go index 5405e093..a88c6e2f 100644 --- a/internal/metadatastorage/sqlstore/store.go +++ b/pkg/metadatastorage/sqlstore/store.go @@ -25,8 +25,8 @@ import ( "github.com/digitorus/timestamp" "github.com/in-toto/archivista/ent" - "github.com/in-toto/archivista/internal/metadatastorage" - "github.com/in-toto/archivista/internal/metadatastorage/parserregistry" + "github.com/in-toto/archivista/pkg/metadatastorage" + "github.com/in-toto/archivista/pkg/metadatastorage/parserregistry" "github.com/in-toto/go-witness/cryptoutil" "github.com/in-toto/go-witness/dsse" "github.com/in-toto/go-witness/intoto" diff --git a/internal/objectstorage/blobstore/minio.go b/pkg/objectstorage/blobstore/minio.go similarity index 100% rename from internal/objectstorage/blobstore/minio.go rename to pkg/objectstorage/blobstore/minio.go diff --git a/internal/objectstorage/filestore/file.go b/pkg/objectstorage/filestore/file.go similarity index 100% rename from internal/objectstorage/filestore/file.go rename to pkg/objectstorage/filestore/file.go diff --git a/internal/objectstorage/filestore/file_test.go b/pkg/objectstorage/filestore/file_test.go similarity index 97% rename from internal/objectstorage/filestore/file_test.go rename to pkg/objectstorage/filestore/file_test.go index 7521edb5..c9974b8d 100644 --- a/internal/objectstorage/filestore/file_test.go +++ b/pkg/objectstorage/filestore/file_test.go @@ -20,7 +20,7 @@ import ( "path/filepath" "testing" - "github.com/in-toto/archivista/internal/objectstorage/filestore" + "github.com/in-toto/archivista/pkg/objectstorage/filestore" "github.com/stretchr/testify/suite" ) diff --git a/internal/server/server.go b/pkg/server/server.go similarity index 99% rename from internal/server/server.go rename to pkg/server/server.go index 165639f5..c250bfbc 100644 --- a/internal/server/server.go +++ b/pkg/server/server.go @@ -35,9 +35,9 @@ import ( "github.com/in-toto/archivista" _ "github.com/in-toto/archivista/docs" "github.com/in-toto/archivista/ent" - "github.com/in-toto/archivista/internal/artifactstore" - "github.com/in-toto/archivista/internal/config" "github.com/in-toto/archivista/pkg/api" + "github.com/in-toto/archivista/pkg/artifactstore" + "github.com/in-toto/archivista/pkg/config" "github.com/sirupsen/logrus" httpSwagger "github.com/swaggo/http-swagger/v2" ) diff --git a/internal/server/server_test.go b/pkg/server/server_test.go similarity index 99% rename from internal/server/server_test.go rename to pkg/server/server_test.go index 420455cb..edca4dfc 100644 --- a/internal/server/server_test.go +++ b/pkg/server/server_test.go @@ -27,9 +27,9 @@ import ( "testing" "github.com/gorilla/mux" - "github.com/in-toto/archivista/internal/artifactstore" - "github.com/in-toto/archivista/internal/config" "github.com/in-toto/archivista/pkg/api" + "github.com/in-toto/archivista/pkg/artifactstore" + "github.com/in-toto/archivista/pkg/config" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) diff --git a/pkg/server/services.go b/pkg/server/services.go new file mode 100644 index 00000000..5255864f --- /dev/null +++ b/pkg/server/services.go @@ -0,0 +1,191 @@ +// Copyright 2024 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A note: this follows a pattern followed by network service mesh. +// The pattern was copied from the Network Service Mesh Project +// and modified for use here. The original code was published under the +// Apache License V2. + +package server + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/in-toto/archivista/pkg/artifactstore" + "github.com/in-toto/archivista/pkg/config" + "github.com/in-toto/archivista/pkg/metadatastorage/sqlstore" + "github.com/in-toto/archivista/pkg/objectstorage/blobstore" + "github.com/in-toto/archivista/pkg/objectstorage/filestore" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/sirupsen/logrus" +) + +// Service is the interface for the Archivista service +type Service interface { + Setup() (Server, error) + GetConfig() *config.Config + GetFileStoreCh() chan error + GetSQLStoreCh() chan error +} + +// ArchivistaService is the implementation of the Archivista service +type ArchivistaService struct { + Ctx context.Context // context for the service + Cfg *config.Config // configuration for the service (if none it uses environment variables) + fileStoreCh <-chan error + sqlStoreCh <-chan error +} + +// Setup Archivista Service +func (a *ArchivistaService) Setup() (*Server, error) { + var ( + level logrus.Level + err error + sqlStore *sqlstore.Store + fileStore StorerGetter + ) + serverOpts := make([]Option, 0) + + startTime := time.Now() + now := time.Now() + if a.Cfg == nil { + + logrus.Infof("executing: get config from environment (time since start: %s)", time.Since(startTime)) + + a.Cfg = new(config.Config) + if err := a.Cfg.Process(); err != nil { + logrus.Fatal(err) + } + level, err = logrus.ParseLevel(a.Cfg.LogLevel) + if err != nil { + logrus.Fatalf("invalid log level %s", a.Cfg.LogLevel) + } + logrus.WithField("duration", time.Since(now)).Infof("completed phase: get config from environment") + } else { + logrus.Infof("executing: load given config (time since start: %s)", time.Since(startTime)) + level, err = logrus.ParseLevel(a.Cfg.LogLevel) + if err != nil { + logrus.Fatalf("invalid log level %s", a.Cfg.LogLevel) + } + logrus.WithField("duration", time.Since(now)).Infof("completed phase: load given config") + } + logrus.SetLevel(level) + + // ******************************************************************************** + logrus.Infof("executing phase: initializing storage clients (time since start: %s)", time.Since(startTime)) + // ******************************************************************************** + now = time.Now() + fileStore, a.fileStoreCh, err = a.initObjectStore() + if err != nil { + logrus.Fatalf("could not create object store: %+v", err) + } + serverOpts = append(serverOpts, WithObjectStore(fileStore)) + + entClient, err := sqlstore.NewEntClient( + a.Cfg.SQLStoreBackend, + a.Cfg.SQLStoreConnectionString, + sqlstore.ClientWithMaxIdleConns(a.Cfg.SQLStoreMaxIdleConnections), + sqlstore.ClientWithMaxOpenConns(a.Cfg.SQLStoreMaxOpenConnections), + sqlstore.ClientWithConnMaxLifetime(a.Cfg.SQLStoreConnectionMaxLifetime), + ) + + if err != nil { + logrus.Fatalf("could not create ent client: %+v", err) + } + + // Continue with the existing setup code for the SQLStore + sqlStore, a.sqlStoreCh, err = sqlstore.New(context.Background(), entClient) + if err != nil { + logrus.Fatalf("error initializing new SQLStore: %+v", err) + } + serverOpts = append(serverOpts, WithMetadataStore(sqlStore)) + + // Add SQL client for ent + sqlClient := sqlStore.GetClient() + serverOpts = append(serverOpts, WithEntSqlClient(sqlClient)) + + // initialize the artifact store + if a.Cfg.EnableArtifactStore { + wds, err := artifactstore.New(artifactstore.WithConfigFile(a.Cfg.ArtifactStoreConfig)) + if err != nil { + logrus.Fatalf("could not create the artifact store: %+v", err) + } + + serverOpts = append(serverOpts, WithArtifactStore(wds)) + } + + // Create the Archivista server with all options + server, err := New(a.Cfg, serverOpts...) + if err != nil { + logrus.Fatalf("could not create archivista server: %+v", err) + } + + // Ensure background processes are managed + go func() { + <-a.sqlStoreCh + <-a.fileStoreCh + }() + + logrus.WithField("duration", time.Since(now)).Infof("completed phase: initializing storage clients") + + return &server, nil +} + +// GetFileStoreCh returns the file store channel +func (a *ArchivistaService) GetFileStoreCh() <-chan error { + return a.fileStoreCh +} + +// GetSQLStoreCh returns the SQL store channel +func (a *ArchivistaService) GetSQLStoreCh() <-chan error { + return a.sqlStoreCh +} + +func (a *ArchivistaService) initObjectStore() (StorerGetter, <-chan error, error) { + switch strings.ToUpper(a.Cfg.StorageBackend) { + case "FILE": + return filestore.New(a.Ctx, a.Cfg.FileDir, a.Cfg.FileServeOn) + + case "BLOB": + var creds *credentials.Credentials + if a.Cfg.BlobStoreCredentialType == "IAM" { + creds = credentials.NewIAM("") + } else if a.Cfg.BlobStoreCredentialType == "ACCESS_KEY" { + creds = credentials.NewStaticV4(a.Cfg.BlobStoreAccessKeyId, a.Cfg.BlobStoreSecretAccessKeyId, "") + } else { + logrus.Fatalf("invalid blob store credential type: %s", a.Cfg.BlobStoreCredentialType) + } + return blobstore.New( + a.Ctx, + a.Cfg.BlobStoreEndpoint, + creds, + a.Cfg.BlobStoreBucketName, + a.Cfg.BlobStoreUseTLS, + ) + + case "": + errCh := make(chan error) + go func() { + <-a.Ctx.Done() + close(errCh) + }() + return nil, errCh, nil + + default: + return nil, nil, fmt.Errorf("unknown storage backend: %s", a.Cfg.StorageBackend) + } +}