Skip to content

Commit

Permalink
Refactoring services (#287)
Browse files Browse the repository at this point in the history
* refactor: remove iternal for Archivista flexibility

Remove the internal in order to enable Archivista source code user to:
- Extend/Modify the existent services (config, storage, etc) within
  their code base

- Allow to use Archivista encapuslated in other services, example API
  service

Signed-off-by: Kairo Araujo <kairo.araujo@testifysec.com>

* refactor: simplify archivista main cmd

Simplifies Archivista main cmd moving all the service logic into the
`pkg/server`.

It allows users to use the Archivista Server instance as an Service
Interface allowing to integrate along to API services.

Signed-off-by: Kairo Araujo <kairo.araujo@testifysec.com>

---------

Signed-off-by: Kairo Araujo <kairo.araujo@testifysec.com>
  • Loading branch information
kairoaraujo authored Jun 4, 2024
1 parent 1e9626e commit 2eab116
Show file tree
Hide file tree
Showing 16 changed files with 213 additions and 121 deletions.
123 changes: 12 additions & 111 deletions cmd/archivista/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ package main

import (
"context"
"fmt"
"net"
"net/http"
"os"
Expand All @@ -32,13 +31,7 @@ import (

nested "github.com/antonfisher/nested-logrus-formatter"
"github.com/gorilla/handlers"
"github.com/in-toto/archivista/internal/artifactstore"
"github.com/in-toto/archivista/internal/config"
"github.com/in-toto/archivista/internal/metadatastorage/sqlstore"
"github.com/in-toto/archivista/internal/objectstorage/blobstore"
"github.com/in-toto/archivista/internal/objectstorage/filestore"
"github.com/in-toto/archivista/internal/server"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/in-toto/archivista/pkg/server"
"github.com/sirupsen/logrus"
)

Expand All @@ -57,76 +50,19 @@ func main() {
defer cancel()

startTime := time.Now()
serverOpts := make([]server.Option, 0)

logrus.Infof("executing phase 1: get config from environment (time since start: %s)", time.Since(startTime))
now := time.Now()

cfg := new(config.Config)
if err := cfg.Process(); err != nil {
logrus.Fatal(err)
}

level, err := logrus.ParseLevel(cfg.LogLevel)
if err != nil {
logrus.Fatalf("invalid log level %s", cfg.LogLevel)
}
logrus.SetLevel(level)

logrus.WithField("duration", time.Since(now)).Infof("completed phase 1: get config from environment")

// ********************************************************************************
logrus.Infof("executing phase 2: initializing storage clients (time since start: %s)", time.Since(startTime))
// ********************************************************************************
now = time.Now()
fileStore, fileStoreCh, err := initObjectStore(ctx, cfg)
if err != nil {
logrus.Fatalf("error initializing storage clients: %+v", err)
}
serverOpts = append(serverOpts, server.WithObjectStore(fileStore))
archivistaService := &server.ArchivistaService{Ctx: ctx, Cfg: nil}

entClient, err := sqlstore.NewEntClient(
cfg.SQLStoreBackend,
cfg.SQLStoreConnectionString,
sqlstore.ClientWithMaxIdleConns(cfg.SQLStoreMaxIdleConnections),
sqlstore.ClientWithMaxOpenConns(cfg.SQLStoreMaxOpenConnections),
sqlstore.ClientWithConnMaxLifetime(cfg.SQLStoreConnectionMaxLifetime))
server, err := archivistaService.Setup()
if err != nil {
logrus.Fatalf("could not create ent client: %+v", err)
logrus.Fatalf("unable to setup archivista service: %+v", err)
}

sqlStore, sqlStoreCh, err := sqlstore.New(ctx, entClient)
if err != nil {
logrus.Fatalf("error initializing mysql client: %+v", err)
}
serverOpts = append(serverOpts, server.WithMetadataStore(sqlStore))

logrus.WithField("duration", time.Since(now)).Infof("completed phase 3: initializing storage clients")

// ********************************************************************************
logrus.Infof("executing phase 3: create and register http service (time since start: %s)", time.Since(startTime))
logrus.Infof("executing phase: create and register http service (time since start: %s)", time.Since(startTime))
// ********************************************************************************
now = time.Now()

// initialize the artifact store
if cfg.EnableArtifactStore {
wds, err := artifactstore.New(artifactstore.WithConfigFile(cfg.ArtifactStoreConfig))
if err != nil {
logrus.Fatalf("could not create the artifact store: %+v", err)
}

serverOpts = append(serverOpts, server.WithArtifactStore(wds))
}

// initialize the server
sqlClient := sqlStore.GetClient()
serverOpts = append(serverOpts, server.WithEntSqlClient(sqlClient))
server, err := server.New(cfg, serverOpts...)
if err != nil {
logrus.Fatalf("could not create archivista server: %+v", err)
}
now := time.Now()

listenAddress := cfg.ListenOn
listenAddress := archivistaService.Cfg.ListenOn
listenAddress = strings.ToLower(strings.TrimSpace(listenAddress))
proto := ""
if strings.HasPrefix(listenAddress, "tcp://") {
Expand All @@ -143,10 +79,10 @@ func main() {
}
srv := &http.Server{
Handler: handlers.CORS(
handlers.AllowedOrigins(cfg.CORSAllowOrigins),
handlers.AllowedOrigins(archivistaService.Cfg.CORSAllowOrigins),
handlers.AllowedMethods([]string{"GET", "POST", "OPTIONS"}),
handlers.AllowedHeaders([]string{"Accept", "Content-Type", "Content-Length", "Accept-Encoding", "X-CSRF-Token", "Authorization"}),
)(server.Router()),
)(server.Router()),
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
}
Expand All @@ -156,47 +92,12 @@ func main() {
}
}()

logrus.WithField("duration", time.Since(now)).Infof("completed phase 5: create and register http service")
logrus.WithField("duration", time.Since(now)).Infof("completed phase: create and register http service")
logrus.Infof("startup complete (time since start: %s)", time.Since(startTime))

<-ctx.Done()
<-fileStoreCh
<-sqlStoreCh
<-archivistaService.GetFileStoreCh()
<-archivistaService.GetSQLStoreCh()

logrus.Infof("exiting, uptime: %v", time.Since(startTime))
}

func initObjectStore(ctx context.Context, cfg *config.Config) (server.StorerGetter, <-chan error, error) {
switch strings.ToUpper(cfg.StorageBackend) {
case "FILE":
return filestore.New(ctx, cfg.FileDir, cfg.FileServeOn)

case "BLOB":
var creds *credentials.Credentials
if cfg.BlobStoreCredentialType == "IAM" {
creds = credentials.NewIAM("")
} else if cfg.BlobStoreCredentialType == "ACCESS_KEY" {
creds = credentials.NewStaticV4(cfg.BlobStoreAccessKeyId, cfg.BlobStoreSecretAccessKeyId, "")
} else {
logrus.Fatalln("invalid blob store credential type: ", cfg.BlobStoreCredentialType)
}
return blobstore.New(
ctx,
cfg.BlobStoreEndpoint,
creds,
cfg.BlobStoreBucketName,
cfg.BlobStoreUseTLS,
)

case "":
errCh := make(chan error)
go func() {
<-ctx.Done()
close(errCh)
}()
return nil, errCh, nil

default:
return nil, nil, fmt.Errorf("unknown storage backend: %s", cfg.StorageBackend)
}
}
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (

"github.com/google/uuid"
"github.com/in-toto/archivista/ent"
"github.com/in-toto/archivista/internal/metadatastorage"
"github.com/in-toto/archivista/pkg/metadatastorage"
"github.com/in-toto/go-witness/attestation"
)

Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
package parserregistry

import (
"github.com/in-toto/archivista/internal/metadatastorage"
"github.com/in-toto/archivista/internal/metadatastorage/attestationcollection"
"github.com/in-toto/archivista/pkg/metadatastorage"
"github.com/in-toto/archivista/pkg/metadatastorage/attestationcollection"
)

var (
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ import (

"github.com/digitorus/timestamp"
"github.com/in-toto/archivista/ent"
"github.com/in-toto/archivista/internal/metadatastorage"
"github.com/in-toto/archivista/internal/metadatastorage/parserregistry"
"github.com/in-toto/archivista/pkg/metadatastorage"
"github.com/in-toto/archivista/pkg/metadatastorage/parserregistry"
"github.com/in-toto/go-witness/cryptoutil"
"github.com/in-toto/go-witness/dsse"
"github.com/in-toto/go-witness/intoto"
Expand Down
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (
"path/filepath"
"testing"

"github.com/in-toto/archivista/internal/objectstorage/filestore"
"github.com/in-toto/archivista/pkg/objectstorage/filestore"
"github.com/stretchr/testify/suite"
)

Expand Down
4 changes: 2 additions & 2 deletions internal/server/server.go → pkg/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@ import (
"github.com/in-toto/archivista"
_ "github.com/in-toto/archivista/docs"
"github.com/in-toto/archivista/ent"
"github.com/in-toto/archivista/internal/artifactstore"
"github.com/in-toto/archivista/internal/config"
"github.com/in-toto/archivista/pkg/api"
"github.com/in-toto/archivista/pkg/artifactstore"
"github.com/in-toto/archivista/pkg/config"
"github.com/sirupsen/logrus"
httpSwagger "github.com/swaggo/http-swagger/v2"
)
Expand Down
4 changes: 2 additions & 2 deletions internal/server/server_test.go → pkg/server/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ import (
"testing"

"github.com/gorilla/mux"
"github.com/in-toto/archivista/internal/artifactstore"
"github.com/in-toto/archivista/internal/config"
"github.com/in-toto/archivista/pkg/api"
"github.com/in-toto/archivista/pkg/artifactstore"
"github.com/in-toto/archivista/pkg/config"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
Expand Down
Loading

0 comments on commit 2eab116

Please sign in to comment.