diff --git a/api/v1/testkube.yaml b/api/v1/testkube.yaml index 97d8dee8a70..2327923163e 100644 --- a/api/v1/testkube.yaml +++ b/api/v1/testkube.yaml @@ -7413,13 +7413,6 @@ components: items: type: string example: ["logline1", "logline2", "logline3"] - executionLogs: - type: object - additionalProperties: - type: array - items: - type: string - example: ["logline1", "logline2", "logline3"] Features: type: object diff --git a/cmd/api-server/commons/commons.go b/cmd/api-server/commons/commons.go new file mode 100644 index 00000000000..1b68a3ed17b --- /dev/null +++ b/cmd/api-server/commons/commons.go @@ -0,0 +1,381 @@ +package commons + +import ( + "context" + "encoding/json" + "fmt" + "net" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/nats-io/nats.go" + "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/mongo" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/types/known/emptypb" + corev1 "k8s.io/api/core/v1" + + "github.com/kubeshop/testkube/internal/config" + dbmigrations "github.com/kubeshop/testkube/internal/db-migrations" + parser "github.com/kubeshop/testkube/internal/template" + "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/cache" + "github.com/kubeshop/testkube/pkg/cloud" + "github.com/kubeshop/testkube/pkg/configmap" + "github.com/kubeshop/testkube/pkg/dbmigrator" + "github.com/kubeshop/testkube/pkg/event" + "github.com/kubeshop/testkube/pkg/event/bus" + "github.com/kubeshop/testkube/pkg/event/kind/slack" + kubeexecutor "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/featureflags" + "github.com/kubeshop/testkube/pkg/imageinspector" + "github.com/kubeshop/testkube/pkg/log" + configRepo "github.com/kubeshop/testkube/pkg/repository/config" + "github.com/kubeshop/testkube/pkg/repository/storage" + "github.com/kubeshop/testkube/pkg/secret" + domainstorage "github.com/kubeshop/testkube/pkg/storage" + "github.com/kubeshop/testkube/pkg/storage/minio" +) + +func ExitOnError(title string, err error) { + if err != nil { + log.DefaultLogger.Errorw(title, "error", err) + os.Exit(1) + } +} + +// General + +func GetEnvironmentVariables() map[string]string { + list := os.Environ() + envs := make(map[string]string, len(list)) + for _, env := range list { + pair := strings.SplitN(env, "=", 2) + if len(pair) != 2 { + continue + } + + envs[pair[0]] += pair[1] + } + return envs +} + +func HandleCancelSignal(ctx context.Context) func() error { + stopSignal := make(chan os.Signal, 1) + signal.Notify(stopSignal, syscall.SIGINT, syscall.SIGTERM) + return func() error { + select { + case <-ctx.Done(): + return nil + case sig := <-stopSignal: + go func() { + <-stopSignal + os.Exit(137) + }() + // Returning an error cancels the errgroup. + return fmt.Errorf("received signal: %v", sig) + } + } +} + +// Configuration + +func MustGetConfig() *config.Config { + cfg, err := config.Get() + ExitOnError("error getting application config", err) + cfg.CleanLegacyVars() + return cfg +} + +func MustGetFeatureFlags() featureflags.FeatureFlags { + features, err := featureflags.Get() + ExitOnError("error getting application feature flags", err) + log.DefaultLogger.Infow("Feature flags configured", "ff", features) + return features +} + +func MustFreePort(port int) { + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + ExitOnError(fmt.Sprintf("Checking if port %d is free", port), err) + _ = ln.Close() + log.DefaultLogger.Debugw("TCP Port is available", "port", port) +} + +func MustGetConfigMapConfig(ctx context.Context, name string, namespace string, defaultTelemetryEnabled bool) *configRepo.ConfigMapConfig { + if name == "" { + name = fmt.Sprintf("testkube-api-server-config-%s", namespace) + } + configMapConfig, err := configRepo.NewConfigMapConfig(name, namespace) + ExitOnError("Getting config map config", err) + + // Load the initial data + err = configMapConfig.Load(ctx, defaultTelemetryEnabled) + if err != nil { + log.DefaultLogger.Warn("error upserting config ConfigMap", "error", err) + } + return configMapConfig +} + +func MustGetMinioClient(cfg *config.Config) domainstorage.Client { + opts := minio.GetTLSOptions(cfg.StorageSSL, cfg.StorageSkipVerify, cfg.StorageCertFile, cfg.StorageKeyFile, cfg.StorageCAFile) + minioClient := minio.NewClient( + cfg.StorageEndpoint, + cfg.StorageAccessKeyID, + cfg.StorageSecretAccessKey, + cfg.StorageRegion, + cfg.StorageToken, + cfg.StorageBucket, + opts..., + ) + err := minioClient.Connect() + ExitOnError("Connecting to minio", err) + if expErr := minioClient.SetExpirationPolicy(cfg.StorageExpiration); expErr != nil { + log.DefaultLogger.Errorw("Error setting expiration policy", "error", expErr) + } + return minioClient +} + +func runMongoMigrations(ctx context.Context, db *mongo.Database) error { + migrationsCollectionName := "__migrations" + activeMigrations, err := dbmigrator.GetDbMigrationsFromFs(dbmigrations.MongoMigrationsFs) + if err != nil { + return errors.Wrap(err, "failed to obtain MongoDB migrations from disk") + } + dbMigrator := dbmigrator.NewDbMigrator(dbmigrator.NewDatabase(db, migrationsCollectionName), activeMigrations) + plan, err := dbMigrator.Plan(ctx) + if err != nil { + return errors.Wrap(err, "failed to plan MongoDB migrations") + } + if plan.Total == 0 { + log.DefaultLogger.Info("No MongoDB migrations to apply.") + } else { + log.DefaultLogger.Info(fmt.Sprintf("Applying MongoDB migrations: %d rollbacks and %d ups.", len(plan.Downs), len(plan.Ups))) + } + err = dbMigrator.Apply(ctx) + return errors.Wrap(err, "failed to apply MongoDB migrations") +} + +func MustGetMongoDatabase(ctx context.Context, cfg *config.Config, secretClient secret.Interface, migrate bool) *mongo.Database { + mongoSSLConfig := getMongoSSLConfig(cfg, secretClient) + db, err := storage.GetMongoDatabase(cfg.APIMongoDSN, cfg.APIMongoDB, cfg.APIMongoDBType, cfg.APIMongoAllowTLS, mongoSSLConfig) + ExitOnError("Getting mongo database", err) + if migrate { + if err = runMongoMigrations(ctx, db); err != nil { + log.DefaultLogger.Warnf("failed to apply MongoDB migrations: %v", err) + } + } + return db +} + +// getMongoSSLConfig builds the necessary SSL connection info from the settings in the environment variables +// and the given secret reference +func getMongoSSLConfig(cfg *config.Config, secretClient secret.Interface) *storage.MongoSSLConfig { + if cfg.APIMongoSSLCert == "" { + return nil + } + + clientCertPath := "/tmp/mongodb.pem" + rootCAPath := "/tmp/mongodb-root-ca.pem" + mongoSSLSecret, err := secretClient.Get(cfg.APIMongoSSLCert) + ExitOnError(fmt.Sprintf("Could not get secret %s for MongoDB connection", cfg.APIMongoSSLCert), err) + + var keyFile, caFile, pass string + var ok bool + if keyFile, ok = mongoSSLSecret[cfg.APIMongoSSLClientFileKey]; !ok { + log.DefaultLogger.Warnf("Could not find sslClientCertificateKeyFile with key %s in secret %s", cfg.APIMongoSSLClientFileKey, cfg.APIMongoSSLCert) + } + if caFile, ok = mongoSSLSecret[cfg.APIMongoSSLCAFileKey]; !ok { + log.DefaultLogger.Warnf("Could not find sslCertificateAuthorityFile with key %s in secret %s", cfg.APIMongoSSLCAFileKey, cfg.APIMongoSSLCert) + } + if pass, ok = mongoSSLSecret[cfg.APIMongoSSLClientFilePass]; !ok { + log.DefaultLogger.Warnf("Could not find sslClientCertificateKeyFilePassword with key %s in secret %s", cfg.APIMongoSSLClientFilePass, cfg.APIMongoSSLCert) + } + + err = os.WriteFile(clientCertPath, []byte(keyFile), 0644) + ExitOnError("Could not place mongodb certificate key file", err) + + err = os.WriteFile(rootCAPath, []byte(caFile), 0644) + ExitOnError("Could not place mongodb ssl ca file: %s", err) + + return &storage.MongoSSLConfig{ + SSLClientCertificateKeyFile: clientCertPath, + SSLClientCertificateKeyFilePassword: pass, + SSLCertificateAuthoritiyFile: rootCAPath, + } +} + +// Actions + +func ReadDefaultExecutors(cfg *config.Config) (executors []testkube.ExecutorDetails, images kubeexecutor.Images, err error) { + rawExecutors, err := parser.LoadConfigFromStringOrFile( + cfg.TestkubeDefaultExecutors, + cfg.TestkubeConfigDir, + "executors.json", + "executors", + ) + if err != nil { + return nil, images, err + } + + if err = json.Unmarshal([]byte(rawExecutors), &executors); err != nil { + return nil, images, err + } + + enabledExecutors, err := parser.LoadConfigFromStringOrFile( + cfg.TestkubeEnabledExecutors, + cfg.TestkubeConfigDir, + "enabledExecutors", + "enabled executors", + ) + if err != nil { + return nil, images, err + } + + // Load internal images + next := make([]testkube.ExecutorDetails, 0) + for i := range executors { + if executors[i].Name == "logs-sidecar" { + images.LogSidecar = executors[i].Executor.Image + continue + } + if executors[i].Name == "init-executor" { + images.Init = executors[i].Executor.Image + continue + } + if executors[i].Name == "scraper-executor" { + images.Scraper = executors[i].Executor.Image + continue + } + if executors[i].Executor == nil { + continue + } + next = append(next, executors[i]) + } + executors = next + + // When there is no executors selected, enable all + if enabledExecutors == "" { + return executors, images, nil + } + + // Filter enabled executors + specifiedExecutors := make(map[string]struct{}) + for _, executor := range strings.Split(enabledExecutors, ",") { + if strings.TrimSpace(executor) == "" { + continue + } + specifiedExecutors[strings.TrimSpace(executor)] = struct{}{} + } + + next = make([]testkube.ExecutorDetails, 0) + for i := range executors { + if _, ok := specifiedExecutors[executors[i].Name]; ok { + next = append(next, executors[i]) + } + } + + return next, images, nil +} + +func ReadProContext(ctx context.Context, cfg *config.Config, grpcClient cloud.TestKubeCloudAPIClient) config.ProContext { + proContext := config.ProContext{ + APIKey: cfg.TestkubeProAPIKey, + URL: cfg.TestkubeProURL, + TLSInsecure: cfg.TestkubeProTLSInsecure, + WorkerCount: cfg.TestkubeProWorkerCount, + LogStreamWorkerCount: cfg.TestkubeProLogStreamWorkerCount, + WorkflowNotificationsWorkerCount: cfg.TestkubeProWorkflowNotificationsWorkerCount, + SkipVerify: cfg.TestkubeProSkipVerify, + EnvID: cfg.TestkubeProEnvID, + OrgID: cfg.TestkubeProOrgID, + Migrate: cfg.TestkubeProMigrate, + ConnectionTimeout: cfg.TestkubeProConnectionTimeout, + DashboardURI: cfg.TestkubeDashboardURI, + } + + if grpcClient == nil { + return proContext + } + + ctx, cancel := context.WithTimeout(ctx, time.Second*3) + md := metadata.Pairs("api-key", cfg.TestkubeProAPIKey) + ctx = metadata.NewOutgoingContext(ctx, md) + defer cancel() + foundProContext, err := grpcClient.GetProContext(ctx, &emptypb.Empty{}) + if err != nil { + log.DefaultLogger.Warnf("cannot fetch pro-context from cloud: %s", err) + return proContext + } + + if proContext.EnvID == "" { + proContext.EnvID = foundProContext.EnvId + } + + if proContext.OrgID == "" { + proContext.OrgID = foundProContext.OrgId + } + + return proContext +} + +func MustCreateSlackLoader(cfg *config.Config, envs map[string]string) *slack.SlackLoader { + slackTemplate, err := parser.LoadConfigFromStringOrFile( + cfg.SlackTemplate, + cfg.TestkubeConfigDir, + "slack-template.json", + "slack template", + ) + ExitOnError("Creating slack loader", err) + + slackConfig, err := parser.LoadConfigFromStringOrFile(cfg.SlackConfig, cfg.TestkubeConfigDir, "slack-config.json", "slack config") + ExitOnError("Creating slack loader", err) + + return slack.NewSlackLoader(slackTemplate, slackConfig, cfg.TestkubeClusterName, cfg.TestkubeDashboardURI, + testkube.AllEventTypes, envs) +} + +func MustCreateNATSConnection(cfg *config.Config) *nats.EncodedConn { + // if embedded NATS server is enabled, we'll replace connection with one to the embedded server + if cfg.NatsEmbedded { + _, nc, err := event.ServerWithConnection(cfg.NatsEmbeddedStoreDir) + ExitOnError("Creating NATS connection", err) + + log.DefaultLogger.Info("Started embedded NATS server") + + conn, err := nats.NewEncodedConn(nc, nats.JSON_ENCODER) + ExitOnError("Creating NATS connection", err) + return conn + } + + conn, err := bus.NewNATSEncodedConnection(bus.ConnectionConfig{ + NatsURI: cfg.NatsURI, + NatsSecure: cfg.NatsSecure, + NatsSkipVerify: cfg.NatsSkipVerify, + NatsCertFile: cfg.NatsCertFile, + NatsKeyFile: cfg.NatsKeyFile, + NatsCAFile: cfg.NatsCAFile, + NatsConnectTimeout: cfg.NatsConnectTimeout, + }) + ExitOnError("Creating NATS connection", err) + return conn +} + +// Components + +func CreateImageInspector(cfg *config.Config, configMapClient configmap.Interface, secretClient secret.Interface) imageinspector.Inspector { + inspectorStorages := []imageinspector.Storage{imageinspector.NewMemoryStorage()} + if cfg.EnableImageDataPersistentCache { + configmapStorage := imageinspector.NewConfigMapStorage(configMapClient, cfg.ImageDataPersistentCacheKey, true) + _ = configmapStorage.CopyTo(context.Background(), inspectorStorages[0].(imageinspector.StorageTransfer)) + inspectorStorages = append(inspectorStorages, configmapStorage) + } + return imageinspector.NewInspector( + cfg.TestkubeRegistry, + imageinspector.NewCraneFetcher(), + imageinspector.NewSecretFetcher(secretClient, cache.NewInMemoryCache[*corev1.Secret](), imageinspector.WithSecretCacheTTL(cfg.TestkubeImageCredentialsCacheTTL)), + inspectorStorages..., + ) +} diff --git a/cmd/api-server/commons/deprecated.go b/cmd/api-server/commons/deprecated.go new file mode 100644 index 00000000000..08554084150 --- /dev/null +++ b/cmd/api-server/commons/deprecated.go @@ -0,0 +1,155 @@ +package commons + +import ( + "context" + + "go.mongodb.org/mongo-driver/mongo" + "google.golang.org/grpc" + "sigs.k8s.io/controller-runtime/pkg/client" + + executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" + templatesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1" + testexecutionsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testexecutions/v1" + testsclientv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" + testsourcesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testsources/v1" + testsuiteexecutionsv1 "github.com/kubeshop/testkube-operator/pkg/client/testsuiteexecutions/v1" + testsuitesclientv3 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v3" + "github.com/kubeshop/testkube/internal/config" + "github.com/kubeshop/testkube/pkg/cloud" + cloudresult "github.com/kubeshop/testkube/pkg/cloud/data/result" + cloudtestresult "github.com/kubeshop/testkube/pkg/cloud/data/testresult" + "github.com/kubeshop/testkube/pkg/featureflags" + "github.com/kubeshop/testkube/pkg/log" + logsclient "github.com/kubeshop/testkube/pkg/logs/client" + "github.com/kubeshop/testkube/pkg/repository/result" + "github.com/kubeshop/testkube/pkg/repository/sequence" + "github.com/kubeshop/testkube/pkg/repository/storage" + "github.com/kubeshop/testkube/pkg/repository/testresult" + domainstorage "github.com/kubeshop/testkube/pkg/storage" +) + +//go:generate mockgen -destination=./mock_deprecatedclients.go -package=commons "github.com/kubeshop/testkube/cmd/api-server/commons" DeprecatedClients +type DeprecatedClients interface { + Executors() executorsclientv1.Interface + Tests() testsclientv3.Interface + TestSuites() testsuitesclientv3.Interface + TestSources() testsourcesclientv1.Interface + TestExecutions() testexecutionsclientv1.Interface + TestSuiteExecutions() testsuiteexecutionsv1.Interface + Templates() templatesclientv1.Interface +} + +type deprecatedClients struct { + executors executorsclientv1.Interface + tests testsclientv3.Interface + testSuites testsuitesclientv3.Interface + testSources testsourcesclientv1.Interface + testExecutions testexecutionsclientv1.Interface + testSuiteExecutions testsuiteexecutionsv1.Interface + templates templatesclientv1.Interface +} + +func (d *deprecatedClients) Executors() executorsclientv1.Interface { + return d.executors +} + +func (d *deprecatedClients) Tests() testsclientv3.Interface { + return d.tests +} + +func (d *deprecatedClients) TestSuites() testsuitesclientv3.Interface { + return d.testSuites +} + +func (d *deprecatedClients) TestSources() testsourcesclientv1.Interface { + return d.testSources +} + +func (d *deprecatedClients) TestExecutions() testexecutionsclientv1.Interface { + return d.testExecutions +} + +func (d *deprecatedClients) TestSuiteExecutions() testsuiteexecutionsv1.Interface { + return d.testSuiteExecutions +} + +func (d *deprecatedClients) Templates() templatesclientv1.Interface { + return d.templates +} + +func CreateDeprecatedClients(kubeClient client.Client, namespace string) DeprecatedClients { + return &deprecatedClients{ + executors: executorsclientv1.NewClient(kubeClient, namespace), + tests: testsclientv3.NewClient(kubeClient, namespace), + testSuites: testsuitesclientv3.NewClient(kubeClient, namespace), + testSources: testsourcesclientv1.NewClient(kubeClient, namespace), + testExecutions: testexecutionsclientv1.NewClient(kubeClient, namespace), + testSuiteExecutions: testsuiteexecutionsv1.NewClient(kubeClient, namespace), + templates: templatesclientv1.NewClient(kubeClient, namespace), + } +} + +//go:generate mockgen -destination=./mock_deprecatedrepositories.go -package=commons "github.com/kubeshop/testkube/cmd/api-server/commons" DeprecatedRepositories +type DeprecatedRepositories interface { + TestResults() result.Repository + TestSuiteResults() testresult.Repository +} + +type deprecatedRepositories struct { + testResults result.Repository + testSuiteResults testresult.Repository +} + +func (d *deprecatedRepositories) TestResults() result.Repository { + return d.testResults +} + +func (d *deprecatedRepositories) TestSuiteResults() testresult.Repository { + return d.testSuiteResults +} + +func CreateDeprecatedRepositoriesForCloud(grpcClient cloud.TestKubeCloudAPIClient, grpcConn *grpc.ClientConn, apiKey string) DeprecatedRepositories { + return &deprecatedRepositories{ + testResults: cloudresult.NewCloudResultRepository(grpcClient, grpcConn, apiKey), + testSuiteResults: cloudtestresult.NewCloudRepository(grpcClient, grpcConn, apiKey), + } +} + +func CreateDeprecatedRepositoriesForMongo(ctx context.Context, cfg *config.Config, db *mongo.Database, logGrpcClient logsclient.StreamGetter, storageClient domainstorage.Client, features featureflags.FeatureFlags) DeprecatedRepositories { + isDocDb := cfg.APIMongoDBType == storage.TypeDocDB + sequenceRepository := sequence.NewMongoRepository(db) + mongoResultsRepository := result.NewMongoRepository(db, cfg.APIMongoAllowDiskUse, isDocDb, result.WithFeatureFlags(features), + result.WithLogsClient(logGrpcClient), result.WithMongoRepositorySequence(sequenceRepository)) + resultsRepository := mongoResultsRepository + testResultsRepository := testresult.NewMongoRepository(db, cfg.APIMongoAllowDiskUse, isDocDb, + testresult.WithMongoRepositorySequence(sequenceRepository)) + + // Init logs storage + if cfg.LogsStorage == "minio" { + if cfg.LogsBucket == "" { + log.DefaultLogger.Error("LOGS_BUCKET env var is not set") + } else if ok, err := storageClient.IsConnectionPossible(ctx); ok && (err == nil) { + log.DefaultLogger.Info("setting minio as logs storage") + mongoResultsRepository.OutputRepository = result.NewMinioOutputRepository(storageClient, mongoResultsRepository.ResultsColl, cfg.LogsBucket) + } else { + log.DefaultLogger.Infow("minio is not available, using default logs storage", "error", err) + } + } + + return &deprecatedRepositories{ + testResults: resultsRepository, + testSuiteResults: testResultsRepository, + } +} + +func MustGetLogsV2Client(cfg *config.Config) logsclient.StreamGetter { + creds, err := logsclient.GetGrpcTransportCredentials(logsclient.GrpcConnectionConfig{ + Secure: cfg.LogServerSecure, + SkipVerify: cfg.LogServerSkipVerify, + CertFile: cfg.LogServerCertFile, + KeyFile: cfg.LogServerKeyFile, + CAFile: cfg.LogServerCAFile, + }) + ExitOnError("Getting log server TLS credentials", err) + return logsclient.NewGrpcClient(cfg.LogServerGrpcAddress, creds) +} diff --git a/cmd/api-server/commons/mock_deprecatedclients.go b/cmd/api-server/commons/mock_deprecatedclients.go new file mode 100644 index 00000000000..3208f5ac5f5 --- /dev/null +++ b/cmd/api-server/commons/mock_deprecatedclients.go @@ -0,0 +1,139 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kubeshop/testkube/cmd/api-server/commons (interfaces: DeprecatedClients) + +// Package commons is a generated GoMock package. +package commons + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + executors "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" + templates "github.com/kubeshop/testkube-operator/pkg/client/templates/v1" + testexecutions "github.com/kubeshop/testkube-operator/pkg/client/testexecutions/v1" + tests "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" + testsources "github.com/kubeshop/testkube-operator/pkg/client/testsources/v1" + testsuiteexecutions "github.com/kubeshop/testkube-operator/pkg/client/testsuiteexecutions/v1" + v3 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v3" +) + +// MockDeprecatedClients is a mock of DeprecatedClients interface. +type MockDeprecatedClients struct { + ctrl *gomock.Controller + recorder *MockDeprecatedClientsMockRecorder +} + +// MockDeprecatedClientsMockRecorder is the mock recorder for MockDeprecatedClients. +type MockDeprecatedClientsMockRecorder struct { + mock *MockDeprecatedClients +} + +// NewMockDeprecatedClients creates a new mock instance. +func NewMockDeprecatedClients(ctrl *gomock.Controller) *MockDeprecatedClients { + mock := &MockDeprecatedClients{ctrl: ctrl} + mock.recorder = &MockDeprecatedClientsMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDeprecatedClients) EXPECT() *MockDeprecatedClientsMockRecorder { + return m.recorder +} + +// Executors mocks base method. +func (m *MockDeprecatedClients) Executors() executors.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Executors") + ret0, _ := ret[0].(executors.Interface) + return ret0 +} + +// Executors indicates an expected call of Executors. +func (mr *MockDeprecatedClientsMockRecorder) Executors() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Executors", reflect.TypeOf((*MockDeprecatedClients)(nil).Executors)) +} + +// Templates mocks base method. +func (m *MockDeprecatedClients) Templates() templates.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Templates") + ret0, _ := ret[0].(templates.Interface) + return ret0 +} + +// Templates indicates an expected call of Templates. +func (mr *MockDeprecatedClientsMockRecorder) Templates() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Templates", reflect.TypeOf((*MockDeprecatedClients)(nil).Templates)) +} + +// TestExecutions mocks base method. +func (m *MockDeprecatedClients) TestExecutions() testexecutions.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TestExecutions") + ret0, _ := ret[0].(testexecutions.Interface) + return ret0 +} + +// TestExecutions indicates an expected call of TestExecutions. +func (mr *MockDeprecatedClientsMockRecorder) TestExecutions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TestExecutions", reflect.TypeOf((*MockDeprecatedClients)(nil).TestExecutions)) +} + +// TestSources mocks base method. +func (m *MockDeprecatedClients) TestSources() testsources.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TestSources") + ret0, _ := ret[0].(testsources.Interface) + return ret0 +} + +// TestSources indicates an expected call of TestSources. +func (mr *MockDeprecatedClientsMockRecorder) TestSources() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TestSources", reflect.TypeOf((*MockDeprecatedClients)(nil).TestSources)) +} + +// TestSuiteExecutions mocks base method. +func (m *MockDeprecatedClients) TestSuiteExecutions() testsuiteexecutions.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TestSuiteExecutions") + ret0, _ := ret[0].(testsuiteexecutions.Interface) + return ret0 +} + +// TestSuiteExecutions indicates an expected call of TestSuiteExecutions. +func (mr *MockDeprecatedClientsMockRecorder) TestSuiteExecutions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TestSuiteExecutions", reflect.TypeOf((*MockDeprecatedClients)(nil).TestSuiteExecutions)) +} + +// TestSuites mocks base method. +func (m *MockDeprecatedClients) TestSuites() v3.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TestSuites") + ret0, _ := ret[0].(v3.Interface) + return ret0 +} + +// TestSuites indicates an expected call of TestSuites. +func (mr *MockDeprecatedClientsMockRecorder) TestSuites() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TestSuites", reflect.TypeOf((*MockDeprecatedClients)(nil).TestSuites)) +} + +// Tests mocks base method. +func (m *MockDeprecatedClients) Tests() tests.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Tests") + ret0, _ := ret[0].(tests.Interface) + return ret0 +} + +// Tests indicates an expected call of Tests. +func (mr *MockDeprecatedClientsMockRecorder) Tests() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Tests", reflect.TypeOf((*MockDeprecatedClients)(nil).Tests)) +} diff --git a/cmd/api-server/commons/mock_deprecatedrepositories.go b/cmd/api-server/commons/mock_deprecatedrepositories.go new file mode 100644 index 00000000000..eb3c7a31567 --- /dev/null +++ b/cmd/api-server/commons/mock_deprecatedrepositories.go @@ -0,0 +1,64 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kubeshop/testkube/cmd/api-server/commons (interfaces: DeprecatedRepositories) + +// Package commons is a generated GoMock package. +package commons + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + result "github.com/kubeshop/testkube/pkg/repository/result" + testresult "github.com/kubeshop/testkube/pkg/repository/testresult" +) + +// MockDeprecatedRepositories is a mock of DeprecatedRepositories interface. +type MockDeprecatedRepositories struct { + ctrl *gomock.Controller + recorder *MockDeprecatedRepositoriesMockRecorder +} + +// MockDeprecatedRepositoriesMockRecorder is the mock recorder for MockDeprecatedRepositories. +type MockDeprecatedRepositoriesMockRecorder struct { + mock *MockDeprecatedRepositories +} + +// NewMockDeprecatedRepositories creates a new mock instance. +func NewMockDeprecatedRepositories(ctrl *gomock.Controller) *MockDeprecatedRepositories { + mock := &MockDeprecatedRepositories{ctrl: ctrl} + mock.recorder = &MockDeprecatedRepositoriesMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDeprecatedRepositories) EXPECT() *MockDeprecatedRepositoriesMockRecorder { + return m.recorder +} + +// TestResults mocks base method. +func (m *MockDeprecatedRepositories) TestResults() result.Repository { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TestResults") + ret0, _ := ret[0].(result.Repository) + return ret0 +} + +// TestResults indicates an expected call of TestResults. +func (mr *MockDeprecatedRepositoriesMockRecorder) TestResults() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TestResults", reflect.TypeOf((*MockDeprecatedRepositories)(nil).TestResults)) +} + +// TestSuiteResults mocks base method. +func (m *MockDeprecatedRepositories) TestSuiteResults() testresult.Repository { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TestSuiteResults") + ret0, _ := ret[0].(testresult.Repository) + return ret0 +} + +// TestSuiteResults indicates an expected call of TestSuiteResults. +func (mr *MockDeprecatedRepositoriesMockRecorder) TestSuiteResults() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TestSuiteResults", reflect.TypeOf((*MockDeprecatedRepositories)(nil).TestSuiteResults)) +} diff --git a/cmd/api-server/main.go b/cmd/api-server/main.go index 11a1ddbc8bb..c0eed718e71 100644 --- a/cmd/api-server/main.go +++ b/cmd/api-server/main.go @@ -2,73 +2,46 @@ package main import ( "context" - "encoding/json" "flag" "fmt" - "net" - "os" - "os/signal" - "path/filepath" - "strings" - "syscall" - "time" - corev1 "k8s.io/api/core/v1" - - "github.com/nats-io/nats.go" - "google.golang.org/grpc/metadata" - "google.golang.org/protobuf/types/known/emptypb" - - "github.com/kubeshop/testkube/pkg/cache" - "github.com/kubeshop/testkube/pkg/testworkflows/executionworker" - "github.com/kubeshop/testkube/pkg/testworkflows/executionworker/kubernetesworker" - "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowconfig" + "github.com/gofiber/fiber/v2/middleware/cors" + "google.golang.org/grpc" executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" + testkubeclientset "github.com/kubeshop/testkube-operator/pkg/clientset/versioned" + "github.com/kubeshop/testkube/cmd/api-server/commons" + "github.com/kubeshop/testkube/cmd/api-server/services" + "github.com/kubeshop/testkube/internal/app/api/debug" + "github.com/kubeshop/testkube/internal/app/api/oauth" + cloudartifacts "github.com/kubeshop/testkube/pkg/cloud/data/artifact" cloudtestworkflow "github.com/kubeshop/testkube/pkg/cloud/data/testworkflow" - "github.com/kubeshop/testkube/pkg/imageinspector" + "github.com/kubeshop/testkube/pkg/event/kind/cdevent" + "github.com/kubeshop/testkube/pkg/event/kind/k8sevent" + "github.com/kubeshop/testkube/pkg/event/kind/webhook" + ws "github.com/kubeshop/testkube/pkg/event/kind/websocket" + oauth2 "github.com/kubeshop/testkube/pkg/oauth" testworkflow2 "github.com/kubeshop/testkube/pkg/repository/testworkflow" "github.com/kubeshop/testkube/pkg/secretmanager" + "github.com/kubeshop/testkube/pkg/server" "github.com/kubeshop/testkube/pkg/tcl/checktcl" "github.com/kubeshop/testkube/pkg/tcl/schedulertcl" + "github.com/kubeshop/testkube/pkg/testworkflows/executionworker/executionworkertypes" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/presets" - "go.mongodb.org/mongo-driver/mongo" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - - cloudartifacts "github.com/kubeshop/testkube/pkg/cloud/data/artifact" - domainstorage "github.com/kubeshop/testkube/pkg/storage" "github.com/kubeshop/testkube/pkg/storage/minio" - "github.com/kubeshop/testkube/pkg/api/v1/testkube" - "github.com/kubeshop/testkube/pkg/event/kind/slack" - - cloudconfig "github.com/kubeshop/testkube/pkg/cloud/data/config" - - cloudresult "github.com/kubeshop/testkube/pkg/cloud/data/result" - cloudtestresult "github.com/kubeshop/testkube/pkg/cloud/data/testresult" - "github.com/kubeshop/testkube/internal/common" - "github.com/kubeshop/testkube/internal/config" - dbmigrations "github.com/kubeshop/testkube/internal/db-migrations" parser "github.com/kubeshop/testkube/internal/template" - "github.com/kubeshop/testkube/pkg/featureflags" + "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/version" - "github.com/kubeshop/testkube/pkg/cloud" - configrepository "github.com/kubeshop/testkube/pkg/repository/config" - "github.com/kubeshop/testkube/pkg/repository/result" - "github.com/kubeshop/testkube/pkg/repository/sequence" - "github.com/kubeshop/testkube/pkg/repository/storage" - "github.com/kubeshop/testkube/pkg/repository/testresult" - "golang.org/x/sync/errgroup" - "github.com/pkg/errors" + "github.com/kubeshop/testkube/pkg/cloud" + "github.com/kubeshop/testkube/pkg/repository/sequence" - "github.com/kubeshop/testkube/internal/app/api/debug" "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/pkg/agent" "github.com/kubeshop/testkube/pkg/event" @@ -79,27 +52,16 @@ import ( logsclient "github.com/kubeshop/testkube/pkg/logs/client" "github.com/kubeshop/testkube/pkg/scheduler" - testkubeclientset "github.com/kubeshop/testkube-operator/pkg/clientset/versioned" "github.com/kubeshop/testkube/pkg/k8sclient" "github.com/kubeshop/testkube/pkg/triggers" kubeclient "github.com/kubeshop/testkube-operator/pkg/client" - scriptsclient "github.com/kubeshop/testkube-operator/pkg/client/scripts/v2" - templatesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1" - testexecutionsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testexecutions/v1" - testsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/tests" - testsclientv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" - testsourcesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testsources/v1" - testsuiteexecutionsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testsuiteexecutions/v1" - testsuitesclientv2 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v2" - testsuitesclientv3 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v3" + testtriggersclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testtriggers/v1" testworkflowsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testworkflows/v1" + deprecatedapiv1 "github.com/kubeshop/testkube/internal/app/api/deprecatedv1" apiv1 "github.com/kubeshop/testkube/internal/app/api/v1" - "github.com/kubeshop/testkube/internal/migrations" "github.com/kubeshop/testkube/pkg/configmap" - "github.com/kubeshop/testkube/pkg/dbmigrator" "github.com/kubeshop/testkube/pkg/log" - "github.com/kubeshop/testkube/pkg/migrator" "github.com/kubeshop/testkube/pkg/reconciler" "github.com/kubeshop/testkube/pkg/secret" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowexecutor" @@ -109,98 +71,101 @@ func init() { flag.Parse() } -func runMigrations() (err error) { - results := migrations.Migrator.GetValidMigrations(version.Version, migrator.MigrationTypeServer) - if len(results) == 0 { - log.DefaultLogger.Debugw("No migrations available for Testkube", "apiVersion", version.Version) - return nil - } +func main() { + cfg := commons.MustGetConfig() + features := commons.MustGetFeatureFlags() - var migrationInfo []string - for _, migration := range results { - migrationInfo = append(migrationInfo, fmt.Sprintf("%+v - %s", migration.Version(), migration.Info())) + // Determine the running mode + mode := common.ModeStandalone + if cfg.TestkubeProAPIKey != "" { + mode = common.ModeAgent } - log.DefaultLogger.Infow("Available migrations for Testkube", "apiVersion", version.Version, "migrations", migrationInfo) - return migrations.Migrator.Run(version.Version, migrator.MigrationTypeServer) -} + // Run services within an errgroup to propagate errors between services. + g, ctx := errgroup.WithContext(context.Background()) -func runMongoMigrations(ctx context.Context, db *mongo.Database, _ string) error { - migrationsCollectionName := "__migrations" - activeMigrations, err := dbmigrator.GetDbMigrationsFromFs(dbmigrations.MongoMigrationsFs) - if err != nil { - return errors.Wrap(err, "failed to obtain MongoDB migrations from disk") - } - dbMigrator := dbmigrator.NewDbMigrator(dbmigrator.NewDatabase(db, migrationsCollectionName), activeMigrations) - plan, err := dbMigrator.Plan(ctx) - if err != nil { - return errors.Wrap(err, "failed to plan MongoDB migrations") - } - if plan.Total == 0 { - log.DefaultLogger.Info("No MongoDB migrations to apply.") - } else { - log.DefaultLogger.Info(fmt.Sprintf("Applying MongoDB migrations: %d rollbacks and %d ups.", len(plan.Downs), len(plan.Ups))) + // Cancel the errgroup context on SIGINT and SIGTERM, + // which shuts everything down gracefully. + g.Go(commons.HandleCancelSignal(ctx)) + + commons.MustFreePort(cfg.APIServerPort) + commons.MustFreePort(cfg.GraphqlPort) + + kubeClient, err := kubeclient.GetClient() + commons.ExitOnError("Getting kubernetes client", err) + + clientset, err := k8sclient.ConnectToK8s() + commons.ExitOnError("Creating k8s clientset", err) + + // k8s + secretClient := secret.NewClientFor(clientset, cfg.TestkubeNamespace) + configMapClient := configmap.NewClientFor(clientset, cfg.TestkubeNamespace) + deprecatedClients := commons.CreateDeprecatedClients(kubeClient, cfg.TestkubeNamespace) + webhooksClient := executorsclientv1.NewWebhooksClient(kubeClient, cfg.TestkubeNamespace) + testTriggersClient := testtriggersclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) + testWorkflowExecutionsClient := testworkflowsclientv1.NewTestWorkflowExecutionsClient(kubeClient, cfg.TestkubeNamespace) + + // TODO: Make granular environment variables, yet backwards compatible + secretConfig := testkube.SecretConfig{ + Prefix: cfg.SecretCreationPrefix, + List: cfg.EnableSecretsEndpoint, + ListAll: cfg.EnableSecretsEndpoint && cfg.EnableListingAllSecrets, + Create: cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation, + Modify: cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation, + Delete: cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation, + AutoCreate: !cfg.DisableSecretCreation, } - err = dbMigrator.Apply(ctx) - return errors.Wrap(err, "failed to apply MongoDB migrations") -} + secretManager := secretmanager.New(clientset, secretConfig) -func main() { - cfg, err := config.Get() - cfg.CleanLegacyVars() - exitOnError("error getting application config", err) + nc := commons.MustCreateNATSConnection(cfg) + eventBus := bus.NewNATSBus(nc) + if cfg.Trace { + eventBus.TraceEvents() + } + eventsEmitter := event.NewEmitter(eventBus, cfg.TestkubeClusterName) - features, err := featureflags.Get() - exitOnError("error getting application feature flags", err) + var logGrpcClient logsclient.StreamGetter + var logsStream logsclient.Stream + if features.LogsV2 { + logGrpcClient = commons.MustGetLogsV2Client(cfg) + logsStream, err = logsclient.NewNatsLogStream(nc.Conn) + commons.ExitOnError("Creating logs streaming client", err) + } - log.DefaultLogger.Infow("Feature flags configured", "ff", features) + configMapConfig := commons.MustGetConfigMapConfig(ctx, cfg.APIServerConfig, cfg.TestkubeNamespace, cfg.TestkubeAnalyticsEnabled) + clusterId, _ := configMapConfig.GetUniqueClusterId(ctx) + telemetryEnabled, _ := configMapConfig.GetTelemetryEnabled(ctx) - // Run services within an errgroup to propagate errors between services. - g, ctx := errgroup.WithContext(context.Background()) + envs := commons.GetEnvironmentVariables() - // Cancel the errgroup context on SIGINT and SIGTERM, - // which shuts everything down gracefully. - stopSignal := make(chan os.Signal, 1) - signal.Notify(stopSignal, syscall.SIGINT, syscall.SIGTERM) - g.Go(func() error { - select { - case <-ctx.Done(): - return nil - case sig := <-stopSignal: - go func() { - <-stopSignal - os.Exit(137) - }() - // Returning an error cancels the errgroup. - return errors.Errorf("received signal: %v", sig) - } - }) + metrics := metrics.NewMetrics() - ln, err := net.Listen("tcp", ":"+cfg.APIServerPort) - exitOnError("Checking if port "+cfg.APIServerPort+"is free", err) - _ = ln.Close() - log.DefaultLogger.Debugw("TCP Port is available", "port", cfg.APIServerPort) + defaultExecutors, images, err := commons.ReadDefaultExecutors(cfg) + commons.ExitOnError("Parsing default executors", err) + if !cfg.TestkubeReadonlyExecutors { + err := kubeexecutor.SyncDefaultExecutors(deprecatedClients.Executors(), cfg.TestkubeNamespace, defaultExecutors) + commons.ExitOnError("Sync default executors", err) + } + jobTemplates, err := parser.ParseJobTemplates(cfg) + commons.ExitOnError("Creating job templates", err) + containerTemplates, err := parser.ParseContainerTemplates(cfg) + commons.ExitOnError("Creating container job templates", err) - ln, err = net.Listen("tcp", ":"+cfg.GraphqlPort) - exitOnError("Checking if port "+cfg.GraphqlPort+"is free", err) - _ = ln.Close() - log.DefaultLogger.Debugw("TCP Port is available", "port", cfg.GraphqlPort) + inspector := commons.CreateImageInspector(cfg, configMapClient, secretClient) - kubeClient, err := kubeclient.GetClient() - exitOnError("Getting kubernetes client", err) + slackLoader := commons.MustCreateSlackLoader(cfg, envs) - secretClient, err := secret.NewClient(cfg.TestkubeNamespace) - exitOnError("Getting secret client", err) + var deprecatedRepositories commons.DeprecatedRepositories + var testWorkflowResultsRepository testworkflow2.Repository + var testWorkflowOutputRepository testworkflow2.OutputRepository + var triggerLeaseBackend triggers.LeaseBackend + var artifactStorage domainstorage.ArtifactsStorage + var storageClient domainstorage.Client + var testWorkflowsClient testworkflowsclientv1.Interface + var testWorkflowTemplatesClient testworkflowsclientv1.TestWorkflowTemplatesInterface - configMapClient, err := configmap.NewClient(cfg.TestkubeNamespace) - exitOnError("Getting config map client", err) - // agent var grpcClient cloud.TestKubeCloudAPIClient var grpcConn *grpc.ClientConn - mode := common.ModeStandalone - if cfg.TestkubeProAPIKey != "" { - mode = common.ModeAgent - } if mode == common.ModeAgent { grpcConn, err = agent.NewGRPCConnection( ctx, @@ -212,79 +177,37 @@ func main() { cfg.TestkubeProCAFile, //nolint log.DefaultLogger, ) - exitOnError("error creating gRPC connection", err) - defer grpcConn.Close() + commons.ExitOnError("error creating gRPC connection", err) grpcClient = cloud.NewTestKubeCloudAPIClient(grpcConn) } - if cfg.EnableDebugServer { - debugSrv := debug.NewDebugServer(cfg.DebugListenAddr) - - g.Go(func() error { - log.DefaultLogger.Infof("starting debug pprof server") - return debugSrv.ListenAndServe() - }) - } + proContext := commons.ReadProContext(ctx, cfg, grpcClient) - // k8s - scriptsClient := scriptsclient.NewClient(kubeClient, cfg.TestkubeNamespace) - testsClientV1 := testsclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) - testsClientV3 := testsclientv3.NewClient(kubeClient, cfg.TestkubeNamespace) - executorsClient := executorsclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) - webhooksClient := executorsclientv1.NewWebhooksClient(kubeClient, cfg.TestkubeNamespace) - testsuitesClientV2 := testsuitesclientv2.NewClient(kubeClient, cfg.TestkubeNamespace) - testsuitesClientV3 := testsuitesclientv3.NewClient(kubeClient, cfg.TestkubeNamespace) - testsourcesClient := testsourcesclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) - testExecutionsClient := testexecutionsclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) - testsuiteExecutionsClient := testsuiteexecutionsclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) - var testWorkflowsClient testworkflowsclientv1.Interface - testWorkflowsClient = testworkflowsclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) - var testWorkflowTemplatesClient testworkflowsclientv1.TestWorkflowTemplatesInterface - testWorkflowTemplatesClient = testworkflowsclientv1.NewTestWorkflowTemplatesClient(kubeClient, cfg.TestkubeNamespace) - testWorkflowExecutionsClient := testworkflowsclientv1.NewTestWorkflowExecutionsClient(kubeClient, cfg.TestkubeNamespace) - templatesClient := templatesclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) - - clientset, err := k8sclient.ConnectToK8s() - if err != nil { - exitOnError("Creating k8s clientset", err) - } + // Check Pro/Enterprise subscription + subscriptionChecker, err := checktcl.NewSubscriptionChecker(ctx, proContext, grpcClient, grpcConn) + commons.ExitOnError("Failed creating subscription checker", err) - k8sCfg, err := k8sclient.GetK8sClientConfig() - if err != nil { - exitOnError("Getting k8s client config", err) + serviceAccountNames := map[string]string{ + cfg.TestkubeNamespace: cfg.JobServiceAccountName, } - testkubeClientset, err := testkubeclientset.NewForConfig(k8sCfg) - if err != nil { - exitOnError("Creating TestKube Clientset", err) + // Pro edition only (tcl protected code) + if cfg.TestkubeExecutionNamespaces != "" { + err = subscriptionChecker.IsActiveOrgPlanEnterpriseForFeature("execution namespace") + commons.ExitOnError("Subscription checking", err) + serviceAccountNames = schedulertcl.GetServiceAccountNamesFromConfig(serviceAccountNames, cfg.TestkubeExecutionNamespaces) } - var logGrpcClient logsclient.StreamGetter - if features.LogsV2 { - creds, err := newGRPCTransportCredentials(cfg) - exitOnError("Getting log server TLS credentials", err) - logGrpcClient = logsclient.NewGrpcClient(cfg.LogServerGrpcAddress, creds) + if mode == common.ModeAgent && cfg.WorkflowStorage == "control-plane" { + testWorkflowsClient = cloudtestworkflow.NewCloudTestWorkflowRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) + testWorkflowTemplatesClient = cloudtestworkflow.NewCloudTestWorkflowTemplateRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) + } else { + testWorkflowsClient = testworkflowsclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) + testWorkflowTemplatesClient = testworkflowsclientv1.NewTestWorkflowTemplatesClient(kubeClient, cfg.TestkubeNamespace) } - // DI - var resultsRepository result.Repository - var testResultsRepository testresult.Repository - var testWorkflowResultsRepository testworkflow2.Repository - var testWorkflowOutputRepository testworkflow2.OutputRepository - var configRepository configrepository.Repository - var triggerLeaseBackend triggers.LeaseBackend - var artifactStorage domainstorage.ArtifactsStorage - var storageClient domainstorage.Client if mode == common.ModeAgent { - resultsRepository = cloudresult.NewCloudResultRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) - testResultsRepository = cloudtestresult.NewCloudRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) - configRepository = cloudconfig.NewCloudResultRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) - - if cfg.WorkflowStorage == "control-plane" { - testWorkflowsClient = cloudtestworkflow.NewCloudTestWorkflowRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) - testWorkflowTemplatesClient = cloudtestworkflow.NewCloudTestWorkflowTemplateRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) - } - // Pro edition only (tcl protected code) + deprecatedRepositories = commons.CreateDeprecatedRepositoriesForCloud(grpcClient, grpcConn, cfg.TestkubeProAPIKey) testWorkflowResultsRepository = cloudtestworkflow.NewCloudRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) var opts []cloudtestworkflow.Option if cfg.StorageSkipVerify { @@ -294,191 +217,35 @@ func main() { triggerLeaseBackend = triggers.NewAcquireAlwaysLeaseBackend() artifactStorage = cloudartifacts.NewCloudArtifactsStorage(grpcClient, grpcConn, cfg.TestkubeProAPIKey) } else { - mongoSSLConfig := getMongoSSLConfig(cfg, secretClient) - db, err := storage.GetMongoDatabase(cfg.APIMongoDSN, cfg.APIMongoDB, cfg.APIMongoDBType, cfg.APIMongoAllowTLS, mongoSSLConfig) - exitOnError("Getting mongo database", err) - isDocDb := cfg.APIMongoDBType == storage.TypeDocDB + // Connect to storages + db := commons.MustGetMongoDatabase(ctx, cfg, secretClient, !cfg.DisableMongoMigrations) + storageClient = commons.MustGetMinioClient(cfg) + + // Build repositories sequenceRepository := sequence.NewMongoRepository(db) - mongoResultsRepository := result.NewMongoRepository(db, cfg.APIMongoAllowDiskUse, isDocDb, result.WithFeatureFlags(features), - result.WithLogsClient(logGrpcClient), result.WithMongoRepositorySequence(sequenceRepository)) - resultsRepository = mongoResultsRepository - testResultsRepository = testresult.NewMongoRepository(db, cfg.APIMongoAllowDiskUse, isDocDb, - testresult.WithMongoRepositorySequence(sequenceRepository)) testWorkflowResultsRepository = testworkflow2.NewMongoRepository(db, cfg.APIMongoAllowDiskUse, testworkflow2.WithMongoRepositorySequence(sequenceRepository)) - configRepository = configrepository.NewMongoRepository(db) triggerLeaseBackend = triggers.NewMongoLeaseBackend(db) - minioClient := newStorageClient(cfg) - if err = minioClient.Connect(); err != nil { - exitOnError("Connecting to minio", err) - } - if expErr := minioClient.SetExpirationPolicy(cfg.StorageExpiration); expErr != nil { - log.DefaultLogger.Errorw("Error setting expiration policy", "error", expErr) - } - storageClient = minioClient testWorkflowOutputRepository = testworkflow2.NewMinioOutputRepository(storageClient, db.Collection(testworkflow2.CollectionName), cfg.LogsBucket) artifactStorage = minio.NewMinIOArtifactClient(storageClient) - // init storage - isMinioStorage := cfg.LogsStorage == "minio" - if isMinioStorage { - bucket := cfg.LogsBucket - if bucket == "" { - log.DefaultLogger.Error("LOGS_BUCKET env var is not set") - } else if ok, err := storageClient.IsConnectionPossible(ctx); ok && (err == nil) { - log.DefaultLogger.Info("setting minio as logs storage") - mongoResultsRepository.OutputRepository = result.NewMinioOutputRepository(storageClient, mongoResultsRepository.ResultsColl, bucket) - } else { - log.DefaultLogger.Infow("minio is not available, using default logs storage", "error", err) - } - } - - // Run DB migrations - if !cfg.DisableMongoMigrations { - err := runMongoMigrations(ctx, db, filepath.Join(cfg.TestkubeConfigDir, "db-migrations")) - if err != nil { - log.DefaultLogger.Warnf("failed to apply MongoDB migrations: %v", err) - } - } - } - - configName := fmt.Sprintf("testkube-api-server-config-%s", cfg.TestkubeNamespace) - if cfg.APIServerConfig != "" { - configName = cfg.APIServerConfig - } - - configMapConfig, err := configrepository.NewConfigMapConfig(configName, cfg.TestkubeNamespace) - exitOnError("Getting config map config", err) - - // try to load from mongo based config first - telemetryEnabled, err := configMapConfig.GetTelemetryEnabled(ctx) - if err != nil { - // fallback to envs in case of failure (no record yet, or other error) - telemetryEnabled = cfg.TestkubeAnalyticsEnabled - } - - var clusterId string - cmConfig, err := configMapConfig.Get(ctx) - if cmConfig.ClusterId != "" { - clusterId = cmConfig.ClusterId - } - - if clusterId == "" { - cmConfig, err = configRepository.Get(ctx) - if err != nil { - log.DefaultLogger.Warnw("error fetching config ConfigMap", "error", err) - } - cmConfig.EnableTelemetry = telemetryEnabled - if cmConfig.ClusterId == "" { - cmConfig.ClusterId, err = configMapConfig.GetUniqueClusterId(ctx) - if err != nil { - log.DefaultLogger.Warnw("error getting unique clusterId", "error", err) - } - } - - clusterId = cmConfig.ClusterId - _, err = configMapConfig.Upsert(ctx, cmConfig) - if err != nil { - log.DefaultLogger.Warn("error upserting config ConfigMap", "error", err) - } - - } - - log.DefaultLogger.Debugw("Getting unique clusterId", "clusterId", clusterId, "error", err) - - // TODO check if this version exists somewhere in stats (probably could be removed) - migrations.Migrator.Add(migrations.NewVersion_0_9_2(scriptsClient, testsClientV1, testsClientV3, testsuitesClientV2)) - if err := runMigrations(); err != nil { - exitOnError("Running server migrations", err) - } - - apiVersion := version.Version - - envs := make(map[string]string) - for _, env := range os.Environ() { - pair := strings.SplitN(env, "=", 2) - if len(pair) != 2 { - continue - } - - envs[pair[0]] += pair[1] - } - - nc, err := newNATSEncodedConnection(cfg) - if err != nil { - exitOnError("Creating NATS connection", err) - } - - eventBus := bus.NewNATSBus(nc) - if cfg.Trace { - eventBus.TraceEvents() - } - - eventsEmitter := event.NewEmitter(eventBus, cfg.TestkubeClusterName) - - var logsStream logsclient.Stream - - if features.LogsV2 { - logsStream, err = logsclient.NewNatsLogStream(nc.Conn) - if err != nil { - exitOnError("Creating logs streaming client", err) - } - } - - metrics := metrics.NewMetrics() - - defaultExecutors, err := parseDefaultExecutors(cfg) - if err != nil { - exitOnError("Parsing default executors", err) - } - - images, err := kubeexecutor.SyncDefaultExecutors(executorsClient, cfg.TestkubeNamespace, defaultExecutors, cfg.TestkubeReadonlyExecutors) - if err != nil { - exitOnError("Sync default executors", err) - } - - jobTemplates, err := parser.ParseJobTemplates(cfg) - if err != nil { - exitOnError("Creating job templates", err) - } - - proContext := newProContext(cfg, grpcClient) - - // Check Pro/Enterprise subscription - var subscriptionChecker checktcl.SubscriptionChecker - if mode == common.ModeAgent { - subscriptionChecker, err = checktcl.NewSubscriptionChecker(ctx, proContext, grpcClient, grpcConn) - exitOnError("Failed creating subscription checker", err) - } - - serviceAccountNames := map[string]string{ - cfg.TestkubeNamespace: cfg.JobServiceAccountName, - } - - // Pro edition only (tcl protected code) - if cfg.TestkubeExecutionNamespaces != "" { - err = subscriptionChecker.IsActiveOrgPlanEnterpriseForFeature("execution namespace") - exitOnError("Subscription checking", err) - - serviceAccountNames = schedulertcl.GetServiceAccountNamesFromConfig(serviceAccountNames, cfg.TestkubeExecutionNamespaces) + deprecatedRepositories = commons.CreateDeprecatedRepositoriesForMongo(ctx, cfg, db, logGrpcClient, storageClient, features) } executor, err := client.NewJobExecutor( - resultsRepository, + deprecatedRepositories, + deprecatedClients, images, jobTemplates, serviceAccountNames, metrics, eventsEmitter, configMapConfig, - testsClientV3, clientset, - testExecutionsClient, - templatesClient, cfg.TestkubeRegistry, cfg.TestkubePodStartTimeout, clusterId, cfg.TestkubeDashboardURI, - "http://"+cfg.APIServerFullname+":"+cfg.APIServerPort, + fmt.Sprintf("http://%s:%d", cfg.APIServerFullname, cfg.APIServerPort), cfg.NatsURI, cfg.Debug, logsStream, @@ -486,30 +253,11 @@ func main() { cfg.TestkubeDefaultStorageClassName, cfg.WhitelistedContainers, ) - if err != nil { - exitOnError("Creating executor client", err) - } - - containerTemplates, err := parser.ParseContainerTemplates(cfg) - if err != nil { - exitOnError("Creating container job templates", err) - } - - inspectorStorages := []imageinspector.Storage{imageinspector.NewMemoryStorage()} - if cfg.EnableImageDataPersistentCache { - configmapStorage := imageinspector.NewConfigMapStorage(configMapClient, cfg.ImageDataPersistentCacheKey, true) - _ = configmapStorage.CopyTo(context.Background(), inspectorStorages[0].(imageinspector.StorageTransfer)) - inspectorStorages = append(inspectorStorages, configmapStorage) - } - inspector := imageinspector.NewInspector( - cfg.TestkubeRegistry, - imageinspector.NewCraneFetcher(), - imageinspector.NewSecretFetcher(secretClient, cache.NewInMemoryCache[*corev1.Secret](), imageinspector.WithSecretCacheTTL(cfg.TestkubeImageCredentialsCacheTTL)), - inspectorStorages..., - ) + commons.ExitOnError("Creating executor client", err) containerExecutor, err := containerexecutor.NewContainerExecutor( - resultsRepository, + deprecatedRepositories, + deprecatedClients, images, containerTemplates, inspector, @@ -517,15 +265,11 @@ func main() { metrics, eventsEmitter, configMapConfig, - executorsClient, - testsClientV3, - testExecutionsClient, - templatesClient, cfg.TestkubeRegistry, cfg.TestkubePodStartTimeout, clusterId, cfg.TestkubeDashboardURI, - "http://"+cfg.APIServerFullname+":"+cfg.APIServerPort, + fmt.Sprintf("http://%s:%d", cfg.APIServerFullname, cfg.APIServerPort), cfg.NatsURI, cfg.Debug, logsStream, @@ -534,26 +278,19 @@ func main() { cfg.WhitelistedContainers, cfg.TestkubeImageCredentialsCacheTTL, ) - if err != nil { - exitOnError("Creating container executor", err) - } + commons.ExitOnError("Creating container executor", err) sched := scheduler.NewScheduler( metrics, executor, containerExecutor, - resultsRepository, - testResultsRepository, - executorsClient, - testsClientV3, - testsuitesClientV3, - testsourcesClient, + deprecatedRepositories, + deprecatedClients, secretClient, eventsEmitter, log.DefaultLogger, configMapConfig, configMapClient, - testsuiteExecutionsClient, eventBus, cfg.TestkubeDashboardURI, features, @@ -561,81 +298,15 @@ func main() { cfg.TestkubeNamespace, cfg.TestkubeProTLSSecret, cfg.TestkubeProRunnerCustomCASecret, + subscriptionChecker, ) - if mode == common.ModeAgent { - sched.WithSubscriptionChecker(subscriptionChecker) - } - - slackLoader, err := newSlackLoader(cfg, envs) - if err != nil { - exitOnError("Creating slack loader", err) - } - - // TODO: Make granular environment variables, yet backwards compatible - secretConfig := testkube.SecretConfig{ - Prefix: cfg.SecretCreationPrefix, - List: cfg.EnableSecretsEndpoint, - ListAll: cfg.EnableSecretsEndpoint && cfg.EnableListingAllSecrets, - Create: cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation, - Modify: cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation, - Delete: cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation, - AutoCreate: !cfg.DisableSecretCreation, - } - - secretManager := secretmanager.New(clientset, secretConfig) + // Build internal execution worker testWorkflowProcessor := presets.NewOpenSource(inspector) if mode == common.ModeAgent { testWorkflowProcessor = presets.NewPro(inspector) } - - // Build internal execution worker - namespacesConfig := map[string]kubernetesworker.NamespaceConfig{} - for n, s := range serviceAccountNames { - namespacesConfig[n] = kubernetesworker.NamespaceConfig{DefaultServiceAccountName: s} - } - cloudUrl := cfg.TestkubeProURL - cloudApiKey := cfg.TestkubeProAPIKey - objectStorageConfig := testworkflowconfig.ObjectStorageConfig{} - if cloudApiKey == "" { - cloudUrl = "" - objectStorageConfig = testworkflowconfig.ObjectStorageConfig{ - Endpoint: cfg.StorageEndpoint, - AccessKeyID: cfg.StorageAccessKeyID, - SecretAccessKey: cfg.StorageSecretAccessKey, - Region: cfg.StorageRegion, - Token: cfg.StorageToken, - Bucket: cfg.StorageBucket, - Ssl: cfg.StorageSSL, - SkipVerify: cfg.StorageSkipVerify, - CertFile: cfg.StorageCertFile, - KeyFile: cfg.StorageKeyFile, - CAFile: cfg.StorageCAFile, - } - } - executionWorker := executionworker.NewKubernetes(clientset, testWorkflowProcessor, kubernetesworker.Config{ - Cluster: kubernetesworker.ClusterConfig{ - Id: clusterId, - DefaultNamespace: cfg.TestkubeNamespace, - DefaultRegistry: cfg.TestkubeRegistry, - Namespaces: namespacesConfig, - }, - ImageInspector: kubernetesworker.ImageInspectorConfig{ - CacheEnabled: cfg.EnableImageDataPersistentCache, - CacheKey: cfg.ImageDataPersistentCacheKey, - CacheTTL: cfg.TestkubeImageCredentialsCacheTTL, - }, - Connection: testworkflowconfig.WorkerConnectionConfig{ - Url: cloudUrl, - ApiKey: cloudApiKey, - SkipVerify: cfg.TestkubeProSkipVerify, - TlsInsecure: cfg.TestkubeProTLSInsecure, - - // TODO: Prepare ControlPlane interface for OSS, so we may unify the communication - LocalApiUrl: fmt.Sprintf("http://%s:%s", cfg.APIServerFullname, cfg.APIServerPort), - ObjectStorage: objectStorageConfig, - }, - }) + executionWorker := services.CreateExecutionWorker(clientset, cfg, clusterId, serviceAccountNames, testWorkflowProcessor) testWorkflowExecutor := testworkflowexecutor.New( eventsEmitter, @@ -653,96 +324,167 @@ func main() { cfg.TestkubeDashboardURI, &proContext, ) + g.Go(func() error { + testWorkflowExecutor.Recover(ctx) + return nil + }) - go testWorkflowExecutor.Recover(context.Background()) + // Initialize event handlers + websocketLoader := ws.NewWebsocketLoader() + eventsEmitter.Loader.Register(webhook.NewWebhookLoader(log.DefaultLogger, webhooksClient, deprecatedClients.Templates(), deprecatedRepositories.TestResults(), deprecatedRepositories.TestSuiteResults(), testWorkflowResultsRepository, metrics, &proContext, envs)) + eventsEmitter.Loader.Register(websocketLoader) + eventsEmitter.Loader.Register(slackLoader) + if cfg.CDEventsTarget != "" { + cdeventLoader, err := cdevent.NewCDEventLoader(cfg.CDEventsTarget, clusterId, cfg.TestkubeNamespace, cfg.TestkubeDashboardURI, testkube.AllEventTypes) + if err == nil { + eventsEmitter.Loader.Register(cdeventLoader) + } else { + log.DefaultLogger.Debugw("cdevents init error", "error", err.Error()) + } + } + if cfg.EnableK8sEvents { + eventsEmitter.Loader.Register(k8sevent.NewK8sEventLoader(clientset, cfg.TestkubeNamespace, testkube.AllEventTypes)) + } + eventsEmitter.Listen(ctx) + g.Go(func() error { + eventsEmitter.Reconcile(ctx) + return nil + }) - api := apiv1.NewTestkubeAPI( + // Create HTTP server + httpServer := server.NewServer(server.Config{Port: cfg.APIServerPort}) + httpServer.Routes.Use(cors.New()) + + // Handle OAuth TODO: deprecated? + httpServer.Routes.Use(oauth.CreateOAuthHandler(oauth.OauthParams{ + ClientID: cfg.TestkubeOAuthClientID, + ClientSecret: cfg.TestkubeOAuthClientSecret, + Provider: oauth2.ProviderType(cfg.TestkubeOAuthProvider), + Scopes: cfg.TestkubeOAuthScopes, + })) + + storageParams := deprecatedapiv1.StorageParams{ + SSL: cfg.StorageSSL, + SkipVerify: cfg.StorageSkipVerify, + CertFile: cfg.StorageCertFile, + KeyFile: cfg.StorageKeyFile, + CAFile: cfg.StorageCAFile, + Endpoint: cfg.StorageEndpoint, + AccessKeyId: cfg.StorageAccessKeyID, + SecretAccessKey: cfg.StorageSecretAccessKey, + Region: cfg.StorageRegion, + Token: cfg.StorageToken, + Bucket: cfg.StorageBucket, + } + deprecatedApi := deprecatedapiv1.NewDeprecatedTestkubeAPI( + deprecatedRepositories, + deprecatedClients, cfg.TestkubeNamespace, - resultsRepository, - testResultsRepository, - testWorkflowResultsRepository, - testWorkflowOutputRepository, - testsClientV3, - executorsClient, - testsuitesClientV3, secretClient, - secretManager, - webhooksClient, - clientset, - testkubeClientset, - testsourcesClient, - testWorkflowsClient, - testWorkflowTemplatesClient, - configMapConfig, - clusterId, eventsEmitter, executor, containerExecutor, - testWorkflowExecutor, - executionWorker, metrics, sched, - slackLoader, cfg.GraphqlPort, artifactStorage, - templatesClient, - cfg.TestkubeDashboardURI, - cfg.TestkubeHelmchartVersion, mode, eventBus, secretConfig, features, logsStream, logGrpcClient, + &proContext, + storageParams, + ) + deprecatedApi.Init(httpServer) + + api := apiv1.NewTestkubeAPI( + deprecatedClients, + clusterId, + cfg.TestkubeNamespace, + testWorkflowResultsRepository, + testWorkflowOutputRepository, + artifactStorage, + webhooksClient, + testTriggersClient, + testWorkflowsClient, + testWorkflowTemplatesClient, + configMapConfig, + secretManager, + secretConfig, + testWorkflowExecutor, + executionWorker, + eventsEmitter, + websocketLoader, + metrics, + &proContext, + features, + cfg.TestkubeDashboardURI, + cfg.TestkubeHelmchartVersion, serviceAccountNames, - envs, cfg.TestkubeDockerImageVersion, ) + api.Init(httpServer) if mode == common.ModeAgent { log.DefaultLogger.Info("starting agent service") - api.WithProContext(&proContext) + getTestWorkflowNotificationsStream := func(ctx context.Context, executionID string) (<-chan testkube.TestWorkflowExecutionNotification, error) { + execution, err := testWorkflowResultsRepository.Get(ctx, executionID) + if err != nil { + return nil, err + } + notifications := executionWorker.Notifications(ctx, execution.Id, executionworkertypes.NotificationsOptions{ + Hints: executionworkertypes.Hints{ + Namespace: execution.Namespace, + Signature: execution.Signature, + ScheduledAt: common.Ptr(execution.ScheduledAt), + }, + }) + if notifications.Err() != nil { + return nil, notifications.Err() + } + return notifications.Channel(), nil + } agentHandle, err := agent.NewAgent( log.DefaultLogger, - api.Mux.Handler(), + httpServer.Mux.Handler(), grpcClient, - api.GetLogsStream, - api.GetTestWorkflowNotificationsStream, + deprecatedApi.GetLogsStream, + getTestWorkflowNotificationsStream, clusterId, cfg.TestkubeClusterName, - envs, features, - proContext, + &proContext, cfg.TestkubeDockerImageVersion, ) - if err != nil { - exitOnError("Starting agent", err) - } + commons.ExitOnError("Starting agent", err) g.Go(func() error { err = agentHandle.Run(ctx) - if err != nil { - exitOnError("Running agent", err) - } + commons.ExitOnError("Running agent", err) return nil }) eventsEmitter.Loader.Register(agentHandle) } - api.Init(cfg.CDEventsTarget, cfg.EnableK8sEvents) if !cfg.DisableTestTriggers { + k8sCfg, err := k8sclient.GetK8sClientConfig() + commons.ExitOnError("Getting k8s client config", err) + testkubeClientset, err := testkubeclientset.NewForConfig(k8sCfg) + commons.ExitOnError("Creating TestKube Clientset", err) + // TODO: Check why this simpler options is not working + //testkubeClientset := testkubeclientset.New(clientset.RESTClient()) + triggerService := triggers.NewService( + deprecatedRepositories, + deprecatedClients, sched, clientset, testkubeClientset, - testsuitesClientV3, - testsClientV3, testWorkflowsClient, - resultsRepository, - testResultsRepository, triggerLeaseBackend, log.DefaultLogger, configMapConfig, - executorsClient, executor, eventBus, metrics, @@ -755,245 +497,55 @@ func main() { triggers.WithDisableSecretCreation(!secretConfig.AutoCreate), ) log.DefaultLogger.Info("starting trigger service") - triggerService.Run(ctx) + g.Go(func() error { + triggerService.Run(ctx) + return nil + }) } else { log.DefaultLogger.Info("test triggers are disabled") } if !cfg.DisableReconciler { - reconcilerClient := reconciler.NewClient(clientset, - resultsRepository, - testResultsRepository, - executorsClient, - log.DefaultLogger) + reconcilerClient := reconciler.NewClient(clientset, deprecatedRepositories, deprecatedClients, log.DefaultLogger) g.Go(func() error { return reconcilerClient.Run(ctx) }) } else { - log.DefaultLogger.Info("reconclier is disabled") + log.DefaultLogger.Info("reconciler is disabled") } // telemetry based functions - telemetryCh := make(chan struct{}) - defer close(telemetryCh) - - api.SendTelemetryStartEvent(ctx, telemetryCh) - api.StartTelemetryHeartbeats(ctx, telemetryCh) + g.Go(func() error { + services.HandleTelemetryHeartbeat(ctx, clusterId, configMapConfig) + return nil + }) log.DefaultLogger.Infow( "starting Testkube API server", "telemetryEnabled", telemetryEnabled, "clusterId", clusterId, "namespace", cfg.TestkubeNamespace, - "version", apiVersion, + "version", version.Version, ) + if cfg.EnableDebugServer { + debugSrv := debug.NewDebugServer(cfg.DebugListenAddr) + + g.Go(func() error { + log.DefaultLogger.Infof("starting debug pprof server") + return debugSrv.ListenAndServe() + }) + } + g.Go(func() error { - return api.Run(ctx) + return httpServer.Run(ctx) }) g.Go(func() error { - return api.RunGraphQLServer(ctx, cfg.GraphqlPort) + return deprecatedApi.RunGraphQLServer(ctx) }) if err := g.Wait(); err != nil { log.DefaultLogger.Fatalf("Testkube is shutting down: %v", err) } } - -func parseDefaultExecutors(cfg *config.Config) (executors []testkube.ExecutorDetails, err error) { - rawExecutors, err := parser.LoadConfigFromStringOrFile( - cfg.TestkubeDefaultExecutors, - cfg.TestkubeConfigDir, - "executors.json", - "executors", - ) - if err != nil { - return nil, err - } - - if err = json.Unmarshal([]byte(rawExecutors), &executors); err != nil { - return nil, err - } - - enabledExecutors, err := parser.LoadConfigFromStringOrFile( - cfg.TestkubeEnabledExecutors, - cfg.TestkubeConfigDir, - "enabledExecutors", - "enabled executors", - ) - if err != nil { - return nil, err - } - - specifiedExecutors := make(map[string]struct{}) - if enabledExecutors != "" { - for _, executor := range strings.Split(enabledExecutors, ",") { - if strings.TrimSpace(executor) == "" { - continue - } - - specifiedExecutors[strings.TrimSpace(executor)] = struct{}{} - } - - for i := len(executors) - 1; i >= 0; i-- { - if _, ok := specifiedExecutors[executors[i].Name]; !ok { - executors = append(executors[:i], executors[i+1:]...) - } - } - } - - return executors, nil -} - -func newNATSEncodedConnection(cfg *config.Config) (*nats.EncodedConn, error) { - // if embedded NATS server is enabled, we'll replace connection with one to the embedded server - if cfg.NatsEmbedded { - _, nc, err := event.ServerWithConnection(cfg.NatsEmbeddedStoreDir) - if err != nil { - return nil, err - } - - log.DefaultLogger.Info("Started embedded NATS server") - - return nats.NewEncodedConn(nc, nats.JSON_ENCODER) - } - - return bus.NewNATSEncodedConnection(bus.ConnectionConfig{ - NatsURI: cfg.NatsURI, - NatsSecure: cfg.NatsSecure, - NatsSkipVerify: cfg.NatsSkipVerify, - NatsCertFile: cfg.NatsCertFile, - NatsKeyFile: cfg.NatsKeyFile, - NatsCAFile: cfg.NatsCAFile, - NatsConnectTimeout: cfg.NatsConnectTimeout, - }) -} - -func newStorageClient(cfg *config.Config) *minio.Client { - opts := minio.GetTLSOptions(cfg.StorageSSL, cfg.StorageSkipVerify, cfg.StorageCertFile, cfg.StorageKeyFile, cfg.StorageCAFile) - return minio.NewClient( - cfg.StorageEndpoint, - cfg.StorageAccessKeyID, - cfg.StorageSecretAccessKey, - cfg.StorageRegion, - cfg.StorageToken, - cfg.StorageBucket, - opts..., - ) -} - -func newSlackLoader(cfg *config.Config, envs map[string]string) (*slack.SlackLoader, error) { - slackTemplate, err := parser.LoadConfigFromStringOrFile( - cfg.SlackTemplate, - cfg.TestkubeConfigDir, - "slack-template.json", - "slack template", - ) - if err != nil { - return nil, err - } - - slackConfig, err := parser.LoadConfigFromStringOrFile(cfg.SlackConfig, cfg.TestkubeConfigDir, "slack-config.json", "slack config") - if err != nil { - return nil, err - } - - return slack.NewSlackLoader(slackTemplate, slackConfig, cfg.TestkubeClusterName, cfg.TestkubeDashboardURI, - testkube.AllEventTypes, envs), nil -} - -// getMongoSSLConfig builds the necessary SSL connection info from the settings in the environment variables -// and the given secret reference -func getMongoSSLConfig(cfg *config.Config, secretClient *secret.Client) *storage.MongoSSLConfig { - if cfg.APIMongoSSLCert == "" { - return nil - } - - clientCertPath := "/tmp/mongodb.pem" - rootCAPath := "/tmp/mongodb-root-ca.pem" - mongoSSLSecret, err := secretClient.Get(cfg.APIMongoSSLCert) - exitOnError(fmt.Sprintf("Could not get secret %s for MongoDB connection", cfg.APIMongoSSLCert), err) - - var keyFile, caFile, pass string - var ok bool - if keyFile, ok = mongoSSLSecret[cfg.APIMongoSSLClientFileKey]; !ok { - log.DefaultLogger.Warnf("Could not find sslClientCertificateKeyFile with key %s in secret %s", cfg.APIMongoSSLClientFileKey, cfg.APIMongoSSLCert) - } - if caFile, ok = mongoSSLSecret[cfg.APIMongoSSLCAFileKey]; !ok { - log.DefaultLogger.Warnf("Could not find sslCertificateAuthorityFile with key %s in secret %s", cfg.APIMongoSSLCAFileKey, cfg.APIMongoSSLCert) - } - if pass, ok = mongoSSLSecret[cfg.APIMongoSSLClientFilePass]; !ok { - log.DefaultLogger.Warnf("Could not find sslClientCertificateKeyFilePassword with key %s in secret %s", cfg.APIMongoSSLClientFilePass, cfg.APIMongoSSLCert) - } - - err = os.WriteFile(clientCertPath, []byte(keyFile), 0644) - exitOnError("Could not place mongodb certificate key file", err) - - err = os.WriteFile(rootCAPath, []byte(caFile), 0644) - exitOnError("Could not place mongodb ssl ca file: %s", err) - - return &storage.MongoSSLConfig{ - SSLClientCertificateKeyFile: clientCertPath, - SSLClientCertificateKeyFilePassword: pass, - SSLCertificateAuthoritiyFile: rootCAPath, - } -} - -func newGRPCTransportCredentials(cfg *config.Config) (credentials.TransportCredentials, error) { - return logsclient.GetGrpcTransportCredentials(logsclient.GrpcConnectionConfig{ - Secure: cfg.LogServerSecure, - SkipVerify: cfg.LogServerSkipVerify, - CertFile: cfg.LogServerCertFile, - KeyFile: cfg.LogServerKeyFile, - CAFile: cfg.LogServerCAFile, - }) -} - -func newProContext(cfg *config.Config, grpcClient cloud.TestKubeCloudAPIClient) config.ProContext { - proContext := config.ProContext{ - APIKey: cfg.TestkubeProAPIKey, - URL: cfg.TestkubeProURL, - TLSInsecure: cfg.TestkubeProTLSInsecure, - WorkerCount: cfg.TestkubeProWorkerCount, - LogStreamWorkerCount: cfg.TestkubeProLogStreamWorkerCount, - WorkflowNotificationsWorkerCount: cfg.TestkubeProWorkflowNotificationsWorkerCount, - SkipVerify: cfg.TestkubeProSkipVerify, - EnvID: cfg.TestkubeProEnvID, - OrgID: cfg.TestkubeProOrgID, - Migrate: cfg.TestkubeProMigrate, - ConnectionTimeout: cfg.TestkubeProConnectionTimeout, - DashboardURI: cfg.TestkubeDashboardURI, - } - - if grpcClient == nil { - return proContext - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - md := metadata.Pairs("api-key", cfg.TestkubeProAPIKey) - ctx = metadata.NewOutgoingContext(ctx, md) - defer cancel() - getProContext, err := grpcClient.GetProContext(ctx, &emptypb.Empty{}) - if err != nil { - log.DefaultLogger.Warnf("cannot fetch pro-context from cloud: %s", err) - return proContext - } - - if proContext.EnvID == "" { - proContext.EnvID = getProContext.EnvId - } - - if proContext.OrgID == "" { - proContext.OrgID = getProContext.OrgId - } - - return proContext -} - -func exitOnError(title string, err error) { - if err != nil { - log.DefaultLogger.Errorw(title, "error", err) - os.Exit(1) - } -} diff --git a/cmd/api-server/services/executionworker.go b/cmd/api-server/services/executionworker.go new file mode 100644 index 00000000000..b78965689cd --- /dev/null +++ b/cmd/api-server/services/executionworker.go @@ -0,0 +1,69 @@ +package services + +import ( + "fmt" + + "k8s.io/client-go/kubernetes" + + "github.com/kubeshop/testkube/internal/config" + "github.com/kubeshop/testkube/pkg/testworkflows/executionworker" + "github.com/kubeshop/testkube/pkg/testworkflows/executionworker/executionworkertypes" + "github.com/kubeshop/testkube/pkg/testworkflows/executionworker/kubernetesworker" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowconfig" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" +) + +func CreateExecutionWorker( + clientSet kubernetes.Interface, + cfg *config.Config, + clusterId string, + serviceAccountNames map[string]string, + processor testworkflowprocessor.Processor, +) executionworkertypes.Worker { + namespacesConfig := map[string]kubernetesworker.NamespaceConfig{} + for n, s := range serviceAccountNames { + namespacesConfig[n] = kubernetesworker.NamespaceConfig{DefaultServiceAccountName: s} + } + cloudUrl := cfg.TestkubeProURL + cloudApiKey := cfg.TestkubeProAPIKey + objectStorageConfig := testworkflowconfig.ObjectStorageConfig{} + if cloudApiKey == "" { + cloudUrl = "" + objectStorageConfig = testworkflowconfig.ObjectStorageConfig{ + Endpoint: cfg.StorageEndpoint, + AccessKeyID: cfg.StorageAccessKeyID, + SecretAccessKey: cfg.StorageSecretAccessKey, + Region: cfg.StorageRegion, + Token: cfg.StorageToken, + Bucket: cfg.StorageBucket, + Ssl: cfg.StorageSSL, + SkipVerify: cfg.StorageSkipVerify, + CertFile: cfg.StorageCertFile, + KeyFile: cfg.StorageKeyFile, + CAFile: cfg.StorageCAFile, + } + } + return executionworker.NewKubernetes(clientSet, processor, kubernetesworker.Config{ + Cluster: kubernetesworker.ClusterConfig{ + Id: clusterId, + DefaultNamespace: cfg.TestkubeNamespace, + DefaultRegistry: cfg.TestkubeRegistry, + Namespaces: namespacesConfig, + }, + ImageInspector: kubernetesworker.ImageInspectorConfig{ + CacheEnabled: cfg.EnableImageDataPersistentCache, + CacheKey: cfg.ImageDataPersistentCacheKey, + CacheTTL: cfg.TestkubeImageCredentialsCacheTTL, + }, + Connection: testworkflowconfig.WorkerConnectionConfig{ + Url: cloudUrl, + ApiKey: cloudApiKey, + SkipVerify: cfg.TestkubeProSkipVerify, + TlsInsecure: cfg.TestkubeProTLSInsecure, + + // TODO: Prepare ControlPlane interface for OSS, so we may unify the communication + LocalApiUrl: fmt.Sprintf("http://%s:%d", cfg.APIServerFullname, cfg.APIServerPort), + ObjectStorage: objectStorageConfig, + }, + }) +} diff --git a/cmd/api-server/services/telemetry.go b/cmd/api-server/services/telemetry.go new file mode 100644 index 00000000000..eb6902859c9 --- /dev/null +++ b/cmd/api-server/services/telemetry.go @@ -0,0 +1,48 @@ +package services + +import ( + "context" + "os" + "time" + + "github.com/kubeshop/testkube/pkg/log" + configRepo "github.com/kubeshop/testkube/pkg/repository/config" + "github.com/kubeshop/testkube/pkg/telemetry" + "github.com/kubeshop/testkube/pkg/utils/text" + "github.com/kubeshop/testkube/pkg/version" +) + +const ( + heartbeatInterval = time.Hour +) + +func HandleTelemetryHeartbeat(ctx context.Context, clusterId string, configMapConfig configRepo.Repository) { + telemetryEnabled, _ := configMapConfig.GetTelemetryEnabled(ctx) + if telemetryEnabled { + out, err := telemetry.SendServerStartEvent(clusterId, version.Version) + if err != nil { + log.DefaultLogger.Debug("telemetry send error", "error", err.Error()) + } else { + log.DefaultLogger.Debugw("sending telemetry server start event", "output", out) + } + } + + ticker := time.NewTicker(heartbeatInterval) + for { + telemetryEnabled, _ = configMapConfig.GetTelemetryEnabled(ctx) + if telemetryEnabled { + l := log.DefaultLogger.With("measurmentId", telemetry.TestkubeMeasurementID, "secret", text.Obfuscate(telemetry.TestkubeMeasurementSecret)) + host, err := os.Hostname() + if err != nil { + l.Debugw("getting hostname error", "hostname", host, "error", err) + } + out, err := telemetry.SendHeartbeatEvent(host, version.Version, clusterId) + if err != nil { + l.Debugw("sending heartbeat telemetry event error", "error", err) + } else { + l.Debugw("sending heartbeat telemetry event", "output", out) + } + } + <-ticker.C + } +} diff --git a/cmd/kubectl-testkube/commands/agent/migrate.go b/cmd/kubectl-testkube/commands/agent/migrate.go index 78fb6a0c7e6..054fb888144 100644 --- a/cmd/kubectl-testkube/commands/agent/migrate.go +++ b/cmd/kubectl-testkube/commands/agent/migrate.go @@ -2,9 +2,6 @@ package agent import ( "github.com/spf13/cobra" - - "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common" - "github.com/kubeshop/testkube/pkg/ui" ) func NewMigrateAgentCmd() *cobra.Command { @@ -13,11 +10,7 @@ func NewMigrateAgentCmd() *cobra.Command { Short: "manual migrate agent command", Long: `migrate agent command will run agent migrations greater or equals current version`, Run: func(cmd *cobra.Command, args []string) { - hasMigrations, err := common.RunAgentMigrations(cmd) - ui.ExitOnError("Running agent migrations", err) - if hasMigrations { - ui.Success("All agent migrations executed successfully") - } + // TODO: Delete, as we don't have any migrations }, } diff --git a/cmd/kubectl-testkube/commands/common/helper.go b/cmd/kubectl-testkube/commands/common/helper.go index 1b2260322b5..8ebeabd977a 100644 --- a/cmd/kubectl-testkube/commands/common/helper.go +++ b/cmd/kubectl-testkube/commands/common/helper.go @@ -18,10 +18,8 @@ import ( "github.com/spf13/cobra" "github.com/kubeshop/testkube/cmd/kubectl-testkube/config" - "github.com/kubeshop/testkube/internal/migrations" cloudclient "github.com/kubeshop/testkube/pkg/cloud/client" "github.com/kubeshop/testkube/pkg/cloudlogin" - "github.com/kubeshop/testkube/pkg/migrator" "github.com/kubeshop/testkube/pkg/process" "github.com/kubeshop/testkube/pkg/ui" ) @@ -403,31 +401,6 @@ func UpdateTokens(cfg config.Data, token, refreshToken string) error { return nil } -func RunAgentMigrations(cmd *cobra.Command) (hasMigrations bool, err error) { - client, _, err := GetClient(cmd) - ui.ExitOnError("getting client", err) - - info, err := client.GetServerInfo() - ui.ExitOnError("getting server info", err) - - if info.Version == "" { - ui.Failf("Can't detect cluster version") - } - - ui.Info("Available agent migrations for", info.Version) - results := migrations.Migrator.GetValidMigrations(info.Version, migrator.MigrationTypeClient) - if len(results) == 0 { - ui.Warn("No agent migrations available for", info.Version) - return false, nil - } - - for _, migration := range results { - fmt.Printf("- %+v - %s\n", migration.Version(), migration.Info()) - } - - return true, migrations.Migrator.Run(info.Version, migrator.MigrationTypeClient) -} - func PopulateOrgAndEnvNames(cfg config.Data, orgId, envId, apiUrl string) (config.Data, error) { if orgId != "" { cfg.CloudContext.OrganizationId = orgId diff --git a/cmd/kubectl-testkube/commands/debug/utils.go b/cmd/kubectl-testkube/commands/debug/utils.go index ef377105086..7df546af80a 100644 --- a/cmd/kubectl-testkube/commands/debug/utils.go +++ b/cmd/kubectl-testkube/commands/debug/utils.go @@ -1,7 +1,6 @@ package debug import ( - "fmt" "os" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common" @@ -46,16 +45,4 @@ func PrintDebugInfo(info testkube.DebugInfo) { ui.Info(l) } ui.NL() - - ui.Info("EXECUTION LOGS") - ui.NL() - for id, logs := range info.ExecutionLogs { - ui.Info(fmt.Sprintf("EXECUTION ID: %s", id)) - ui.NL() - for _, l := range logs { - ui.Info(l) - } - ui.NL() - } - ui.NL() } diff --git a/cmd/kubectl-testkube/commands/github/issue_test.go b/cmd/kubectl-testkube/commands/github/issue_test.go index aed6f5098f0..7792efac1f6 100644 --- a/cmd/kubectl-testkube/commands/github/issue_test.go +++ b/cmd/kubectl-testkube/commands/github/issue_test.go @@ -27,10 +27,6 @@ func TestBuildTicket(t *testing.T) { ClusterVersion: "v2.test", ApiLogs: []string{"api logline1", "api logline2"}, OperatorLogs: []string{"operator logline1", "operator logline2", "operator logline3"}, - ExecutionLogs: map[string][]string{ - "execution1": {"execution logline1"}, - "execution2": {"execution logline1", "execution logline2"}, - }, }, wantTitle: "New bug report", wantBody: ` diff --git a/cmd/kubectl-testkube/commands/upgrade.go b/cmd/kubectl-testkube/commands/upgrade.go index 9a1ab0387c5..b290e771c2e 100644 --- a/cmd/kubectl-testkube/commands/upgrade.go +++ b/cmd/kubectl-testkube/commands/upgrade.go @@ -90,11 +90,6 @@ func NewUpgradeCmd() *cobra.Command { ui.ExitOnError("Storing agent data in context", err) } else { ui.Info("Updating Testkube") - hasMigrations, err := common.RunAgentMigrations(cmd) - ui.ExitOnError("Running agent migrations", err) - if hasMigrations { - ui.Success("All agent migrations executed successfully") - } if cliErr := common.HelmUpgradeOrInstallTestkube(options); cliErr != nil { cliErr.Print() diff --git a/go.mod b/go.mod index 9bc0985e7c7..7c228faa479 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/kelseyhightower/envconfig v1.4.0 github.com/kubepug/kubepug v1.7.1 - github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240916120302-48b0ceed8c5d + github.com/kubeshop/testkube-operator v1.17.55-0.20241018115850-515e6a3a0a33 github.com/minio/minio-go/v7 v7.0.47 github.com/montanaflynn/stats v0.6.6 github.com/moogar0880/problems v0.1.1 diff --git a/go.sum b/go.sum index bdae9af276b..671b4cd184a 100644 --- a/go.sum +++ b/go.sum @@ -393,8 +393,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubepug/kubepug v1.7.1 h1:LKhfSxS8Y5mXs50v+3Lpyec+cogErDLcV7CMUuiaisw= github.com/kubepug/kubepug v1.7.1/go.mod h1:lv+HxD0oTFL7ZWjj0u6HKhMbbTIId3eG7aWIW0gyF8g= -github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240916120302-48b0ceed8c5d h1:ZwsRzKUJ+pMC2pF47L/O06Vq8vOYZe17GO3N2rL9lCk= -github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240916120302-48b0ceed8c5d/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk= +github.com/kubeshop/testkube-operator v1.17.55-0.20241018115850-515e6a3a0a33 h1:TFaiCAiKmACKKp2TFUKiuAEc+7CUafUfms5BqGELrnA= +github.com/kubeshop/testkube-operator v1.17.55-0.20241018115850-515e6a3a0a33/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4= diff --git a/internal/app/api/apiutils/utils.go b/internal/app/api/apiutils/utils.go new file mode 100644 index 00000000000..50f44690ea6 --- /dev/null +++ b/internal/app/api/apiutils/utils.go @@ -0,0 +1,76 @@ +package apiutils + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/gofiber/fiber/v2" + "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/mongo" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/kubeshop/testkube/pkg/log" + "github.com/kubeshop/testkube/pkg/problem" + "github.com/kubeshop/testkube/pkg/secretmanager" +) + +const ( + // mediaTypeYAML is yaml media type + mediaTypeYAML = "text/yaml" +) + +func SendLegacyCRDs(c *fiber.Ctx, data string, err error) error { + if err != nil { + return SendError(c, http.StatusBadRequest, fmt.Errorf("could not build CRD: %w", err)) + } + + c.Context().SetContentType(mediaTypeYAML) + return c.SendString(data) +} + +func IsNotFound(err error) bool { + if err == nil { + return false + } + if errors.Is(err, mongo.ErrNoDocuments) || k8serrors.IsNotFound(err) || errors.Is(err, secretmanager.ErrNotFound) { + return true + } + if e, ok := status.FromError(err); ok { + return e.Code() == codes.NotFound + } + return false +} + +// SendWarn writes RFC-7807 json problem to response +func SendWarn(c *fiber.Ctx, status int, err error, context ...interface{}) error { + c.Status(status) + c.Response().Header.Set("Content-Type", "application/problem+json") + log.DefaultLogger.Warnw(err.Error(), "status", status) + pr := problem.New(status, getProblemMessage(err, context)) + return c.JSON(pr) +} + +// SendError writes RFC-7807 json problem to response +func SendError(c *fiber.Ctx, status int, err error, context ...interface{}) error { + c.Status(status) + c.Response().Header.Set("Content-Type", "application/problem+json") + log.DefaultLogger.Errorw(err.Error(), "status", status) + pr := problem.New(status, getProblemMessage(err, context)) + return c.JSON(pr) +} + +// getProblemMessage creates new JSON based problem message and returns it as string +func getProblemMessage(err error, context ...interface{}) string { + message := err.Error() + if len(context) > 0 { + b, err := json.Marshal(context[0]) + if err == nil { + message += ", context: " + string(b) + } + } + + return message +} diff --git a/internal/app/api/v1/executions.go b/internal/app/api/deprecatedv1/executions.go similarity index 81% rename from internal/app/api/v1/executions.go rename to internal/app/api/deprecatedv1/executions.go index 8cab531e84c..cba107426f0 100644 --- a/internal/app/api/v1/executions.go +++ b/internal/app/api/deprecatedv1/executions.go @@ -1,4 +1,4 @@ -package v1 +package deprecatedv1 import ( "bufio" @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" + "github.com/kubeshop/testkube/pkg/datefilter" "github.com/kubeshop/testkube/pkg/logs/events" "github.com/kubeshop/testkube/pkg/repository/result" @@ -41,7 +42,7 @@ const ( ) // ExecuteTestsHandler calls particular executor based on execution request content and type -func (s *TestkubeAPI) ExecuteTestsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ExecuteTestsHandler() fiber.Handler { return func(c *fiber.Ctx) error { ctx := c.Context() errPrefix := "failed to execute test" @@ -56,7 +57,7 @@ func (s *TestkubeAPI) ExecuteTestsHandler() fiber.Handler { var tests []testsv3.Test if id != "" { - test, err := s.TestsClient.Get(id) + test, err := s.DeprecatedClients.Tests().Get(id) if err != nil { if errors.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: client found no test: %w", errPrefix, err)) @@ -66,7 +67,7 @@ func (s *TestkubeAPI) ExecuteTestsHandler() fiber.Handler { tests = append(tests, *test) } else { - testList, err := s.TestsClient.List(c.Query("selector")) + testList, err := s.DeprecatedClients.Tests().List(c.Query("selector")) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: can't get tests: %w", errPrefix, err)) } @@ -110,7 +111,7 @@ func (s *TestkubeAPI) ExecuteTestsHandler() fiber.Handler { } // ListExecutionsHandler returns array of available test executions -func (s *TestkubeAPI) ListExecutionsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ListExecutionsHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to list executions" // TODO refactor into some Services (based on some abstraction for CRDs at least / CRUD) @@ -120,7 +121,7 @@ func (s *TestkubeAPI) ListExecutionsHandler() fiber.Handler { filter := getFilterFromRequest(c) - executions, err := s.ExecutionResults.GetExecutions(c.Context(), filter) + executions, err := s.DeprecatedRepositories.TestResults().GetExecutions(c.Context(), filter) if err != nil { if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: db found no execution results: %w", errPrefix, err)) @@ -128,7 +129,7 @@ func (s *TestkubeAPI) ListExecutionsHandler() fiber.Handler { return s.Error(c, http.StatusInternalServerError, fmt.Errorf("%s: db client failed to get execution results: %w", errPrefix, err)) } - executionTotals, err := s.ExecutionResults.GetExecutionTotals(c.Context(), false, filter) + executionTotals, err := s.DeprecatedRepositories.TestResults().GetExecutionTotals(c.Context(), false, filter) if err != nil { if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: db client found no total execution results: %w", errPrefix, err)) @@ -136,7 +137,7 @@ func (s *TestkubeAPI) ListExecutionsHandler() fiber.Handler { return s.Error(c, http.StatusInternalServerError, fmt.Errorf("%s: db client failed to get total execution results: %w", errPrefix, err)) } - filteredTotals, err := s.ExecutionResults.GetExecutionTotals(c.Context(), true, filter) + filteredTotals, err := s.DeprecatedRepositories.TestResults().GetExecutionTotals(c.Context(), true, filter) if err != nil { if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: db found no total filtered execution results: %w", errPrefix, err)) @@ -153,8 +154,8 @@ func (s *TestkubeAPI) ListExecutionsHandler() fiber.Handler { } } -func (s *TestkubeAPI) GetLogsStream(ctx context.Context, executionID string) (chan output.Output, error) { - execution, err := s.ExecutionResults.Get(ctx, executionID) +func (s *DeprecatedTestkubeAPI) GetLogsStream(ctx context.Context, executionID string) (chan output.Output, error) { + execution, err := s.DeprecatedRepositories.TestResults().Get(ctx, executionID) if err != nil { return nil, fmt.Errorf("can't find execution %s: %w", executionID, err) } @@ -171,7 +172,7 @@ func (s *TestkubeAPI) GetLogsStream(ctx context.Context, executionID string) (ch return logs, nil } -func (s *TestkubeAPI) ExecutionLogsStreamHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ExecutionLogsStreamHandler() fiber.Handler { return websocket.New(func(c *websocket.Conn) { if s.featureFlags.LogsV2 { return @@ -196,7 +197,7 @@ func (s *TestkubeAPI) ExecutionLogsStreamHandler() fiber.Handler { }) } -func (s *TestkubeAPI) ExecutionLogsStreamHandlerV2() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ExecutionLogsStreamHandlerV2() fiber.Handler { return websocket.New(func(c *websocket.Conn) { if !s.featureFlags.LogsV2 { return @@ -231,7 +232,7 @@ func (s *TestkubeAPI) ExecutionLogsStreamHandlerV2() fiber.Handler { } // ExecutionLogsHandler streams the logs from a test execution -func (s *TestkubeAPI) ExecutionLogsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ExecutionLogsHandler() fiber.Handler { return func(c *fiber.Ctx) error { if s.featureFlags.LogsV2 { return nil @@ -252,7 +253,7 @@ func (s *TestkubeAPI) ExecutionLogsHandler() fiber.Handler { s.Log.Debug("start streaming logs") _ = w.Flush() - execution, err := s.ExecutionResults.Get(ctx, executionID) + execution, err := s.DeprecatedRepositories.TestResults().Get(ctx, executionID) if err != nil { output.PrintError(os.Stdout, fmt.Errorf("could not get execution result for ID %s: %w", executionID, err)) s.Log.Errorw("getting execution error", "error", err) @@ -278,7 +279,7 @@ func (s *TestkubeAPI) ExecutionLogsHandler() fiber.Handler { } // ExecutionLogsHandlerV2 streams the logs from a test execution version 2 -func (s *TestkubeAPI) ExecutionLogsHandlerV2() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ExecutionLogsHandlerV2() fiber.Handler { return func(c *fiber.Ctx) error { if !s.featureFlags.LogsV2 { return nil @@ -313,7 +314,7 @@ func (s *TestkubeAPI) ExecutionLogsHandlerV2() fiber.Handler { } // GetExecutionHandler returns test execution object for given test and execution id/name -func (s *TestkubeAPI) GetExecutionHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) GetExecutionHandler() fiber.Handler { return func(c *fiber.Ctx) error { ctx := c.Context() id := c.Params("id", "") @@ -323,7 +324,7 @@ func (s *TestkubeAPI) GetExecutionHandler() fiber.Handler { var err error if id == "" { - execution, err = s.ExecutionResults.Get(ctx, executionID) + execution, err = s.DeprecatedRepositories.TestResults().Get(ctx, executionID) if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("execution %s not found (test:%s)", executionID, id)) } @@ -331,7 +332,7 @@ func (s *TestkubeAPI) GetExecutionHandler() fiber.Handler { return s.Error(c, http.StatusInternalServerError, fmt.Errorf("db client was unable to get execution %s (test:%s): %w", executionID, id, err)) } } else { - execution, err = s.ExecutionResults.GetByNameAndTest(ctx, executionID, id) + execution, err = s.DeprecatedRepositories.TestResults().GetByNameAndTest(ctx, executionID, id) if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("test %s not found for execution %s", id, executionID)) } @@ -344,7 +345,7 @@ func (s *TestkubeAPI) GetExecutionHandler() fiber.Handler { testSecretMap := make(map[string]string) if execution.TestSecretUUID != "" { - testSecretMap, err = s.TestsClient.GetSecretTestVars(execution.TestName, execution.TestSecretUUID) + testSecretMap, err = s.DeprecatedClients.Tests().GetSecretTestVars(execution.TestName, execution.TestSecretUUID) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("client was unable to get test secrets: %w", err)) } @@ -352,7 +353,7 @@ func (s *TestkubeAPI) GetExecutionHandler() fiber.Handler { testSuiteSecretMap := make(map[string]string) if execution.TestSuiteSecretUUID != "" { - testSuiteSecretMap, err = s.TestsSuitesClient.GetSecretTestSuiteVars(execution.TestSuiteName, execution.TestSuiteSecretUUID) + testSuiteSecretMap, err = s.DeprecatedClients.TestSuites().GetSecretTestSuiteVars(execution.TestSuiteName, execution.TestSuiteSecretUUID) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("client was unable to get test suite secrets: %w", err)) } @@ -376,14 +377,14 @@ func (s *TestkubeAPI) GetExecutionHandler() fiber.Handler { } } -func (s *TestkubeAPI) AbortExecutionHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) AbortExecutionHandler() fiber.Handler { return func(c *fiber.Ctx) error { ctx := c.Context() executionID := c.Params("executionID") errPrefix := "failed to abort execution %s" s.Log.Debugw("aborting execution", "executionID", executionID) - execution, err := s.ExecutionResults.Get(ctx, executionID) + execution, err := s.DeprecatedRepositories.TestResults().Get(ctx, executionID) if err != nil { if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: test with execution id %s not found", errPrefix, executionID)) @@ -401,7 +402,7 @@ func (s *TestkubeAPI) AbortExecutionHandler() fiber.Handler { } } -func (s *TestkubeAPI) GetArtifactHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) GetArtifactHandler() fiber.Handler { return func(c *fiber.Ctx) error { executionID := c.Params("executionID") fileName := c.Params("filename") @@ -421,7 +422,7 @@ func (s *TestkubeAPI) GetArtifactHandler() fiber.Handler { //// quickfix end - execution, err := s.ExecutionResults.Get(c.Context(), executionID) + execution, err := s.DeprecatedRepositories.TestResults().Get(c.Context(), executionID) if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: test with execution id/name %s not found", errPrefix, executionID)) } @@ -458,7 +459,7 @@ func (s *TestkubeAPI) GetArtifactHandler() fiber.Handler { } // GetArtifactArchiveHandler returns artifact archive -func (s *TestkubeAPI) GetArtifactArchiveHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) GetArtifactArchiveHandler() fiber.Handler { return func(c *fiber.Ctx) error { executionID := c.Params("executionID") query := c.Request().URI().QueryString() @@ -469,7 +470,7 @@ func (s *TestkubeAPI) GetArtifactArchiveHandler() fiber.Handler { return s.Error(c, http.StatusBadRequest, fmt.Errorf("%s: could not parse query string: %w", errPrefix, err)) } - execution, err := s.ExecutionResults.Get(c.Context(), executionID) + execution, err := s.DeprecatedRepositories.TestResults().Get(c.Context(), executionID) if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: test with execution id/name %s not found", errPrefix, executionID)) } @@ -506,13 +507,13 @@ func (s *TestkubeAPI) GetArtifactArchiveHandler() fiber.Handler { } // ListArtifactsHandler returns list of files in the given bucket -func (s *TestkubeAPI) ListArtifactsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ListArtifactsHandler() fiber.Handler { return func(c *fiber.Ctx) error { executionID := c.Params("executionID") errPrefix := fmt.Sprintf("failed to list artifacts for execution %s", executionID) - execution, err := s.ExecutionResults.Get(c.Context(), executionID) + execution, err := s.DeprecatedRepositories.TestResults().Get(c.Context(), executionID) if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: test with execution id/name %s not found", errPrefix, executionID)) } @@ -548,7 +549,7 @@ func (s *TestkubeAPI) ListArtifactsHandler() fiber.Handler { } // streamLogsFromResult writes logs from the output of executionResult to the writer -func (s *TestkubeAPI) streamLogsFromResult(executionResult *testkube.ExecutionResult, w *bufio.Writer) error { +func (s *DeprecatedTestkubeAPI) streamLogsFromResult(executionResult *testkube.ExecutionResult, w *bufio.Writer) error { enc := json.NewEncoder(w) _, _ = fmt.Fprintf(w, "data: ") s.Log.Debug("using logs from result") @@ -573,7 +574,7 @@ func (s *TestkubeAPI) streamLogsFromResult(executionResult *testkube.ExecutionRe } // streamLogsFromJob streams logs in chunks to writer from the running execution -func (s *TestkubeAPI) streamLogsFromJob(ctx context.Context, executionID, testType, namespace string, w *bufio.Writer) { +func (s *DeprecatedTestkubeAPI) streamLogsFromJob(ctx context.Context, executionID, testType, namespace string, w *bufio.Writer) { enc := json.NewEncoder(w) s.Log.Debugw("getting logs from Kubernetes job") @@ -633,7 +634,7 @@ func mapExecutionsToExecutionSummary(executions []testkube.Execution) []testkube } // GetLatestExecutionLogs returns the latest executions' logs -func (s *TestkubeAPI) GetLatestExecutionLogs(ctx context.Context) (map[string][]string, error) { +func (s *DeprecatedTestkubeAPI) GetLatestExecutionLogs(ctx context.Context) (map[string][]string, error) { latestExecutions, err := s.getNewestExecutions(ctx) if err != nil { return nil, fmt.Errorf("could not list executions: %w", err) @@ -652,9 +653,9 @@ func (s *TestkubeAPI) GetLatestExecutionLogs(ctx context.Context) (map[string][] } // getNewestExecutions returns the latest Testkube executions -func (s *TestkubeAPI) getNewestExecutions(ctx context.Context) ([]testkube.Execution, error) { +func (s *DeprecatedTestkubeAPI) getNewestExecutions(ctx context.Context) ([]testkube.Execution, error) { f := result.NewExecutionsFilter().WithPage(1).WithPageSize(latestExecutions) - executions, err := s.ExecutionResults.GetExecutions(ctx, f) + executions, err := s.DeprecatedRepositories.TestResults().GetExecutions(ctx, f) if err != nil { return []testkube.Execution{}, fmt.Errorf("could not get executions from repo: %w", err) } @@ -662,7 +663,7 @@ func (s *TestkubeAPI) getNewestExecutions(ctx context.Context) ([]testkube.Execu } // getExecutionLogs returns logs from an execution -func (s *TestkubeAPI) getExecutionLogs(ctx context.Context, execution testkube.Execution) ([]string, error) { +func (s *DeprecatedTestkubeAPI) getExecutionLogs(ctx context.Context, execution testkube.Execution) ([]string, error) { var res []string if s.featureFlags.LogsV2 { @@ -699,8 +700,8 @@ func (s *TestkubeAPI) getExecutionLogs(ctx context.Context, execution testkube.E return res, nil } -func (s *TestkubeAPI) getExecutorByTestType(testType string) (client.Executor, error) { - executorCR, err := s.ExecutorsClient.GetByType(testType) +func (s *DeprecatedTestkubeAPI) getExecutorByTestType(testType string) (client.Executor, error) { + executorCR, err := s.DeprecatedClients.Executors().GetByType(testType) if err != nil { return nil, fmt.Errorf("can't get executor spec: %w", err) } @@ -712,7 +713,7 @@ func (s *TestkubeAPI) getExecutorByTestType(testType string) (client.Executor, e } } -func (s *TestkubeAPI) getArtifactStorage(bucket string) (storage.ArtifactsStorage, error) { +func (s *DeprecatedTestkubeAPI) getArtifactStorage(bucket string) (storage.ArtifactsStorage, error) { if s.mode == common.ModeAgent { return s.ArtifactsStorage, nil } @@ -735,7 +736,7 @@ func (s *TestkubeAPI) getArtifactStorage(bucket string) (storage.ArtifactsStorag } // streamLogsFromLogServer writes logs from the output of log server to the writer -func (s *TestkubeAPI) streamLogsFromLogServer(logs chan events.LogResponse, w *bufio.Writer) { +func (s *DeprecatedTestkubeAPI) streamLogsFromLogServer(logs chan events.LogResponse, w *bufio.Writer) { enc := json.NewEncoder(w) s.Log.Debugw("looping through logs channel") // loop through grpc server log lines - it's blocking channel @@ -759,3 +760,67 @@ func (s *TestkubeAPI) streamLogsFromLogServer(logs chan events.LogResponse, w *b s.Log.Debugw("logs streaming stopped") } + +// TODO should we use single generic filter for all list based resources ? +// currently filters for e.g. tests are done "by hand" +func getFilterFromRequest(c *fiber.Ctx) result.Filter { + + filter := result.NewExecutionsFilter() + + // id for /tests/ID/executions + testName := c.Params("id", "") + if testName == "" { + // query param for /executions?testName + testName = c.Query("testName", "") + } + + if testName != "" { + filter = filter.WithTestName(testName) + } + + textSearch := c.Query("textSearch", "") + if textSearch != "" { + filter = filter.WithTextSearch(textSearch) + } + + page, err := strconv.Atoi(c.Query("page", "")) + if err == nil { + filter = filter.WithPage(page) + } + + pageSize, err := strconv.Atoi(c.Query("pageSize", "")) + if err == nil && pageSize != 0 { + filter = filter.WithPageSize(pageSize) + } + + status := c.Query("status", "") + if status != "" { + filter = filter.WithStatus(status) + } + + objectType := c.Query("type", "") + if objectType != "" { + filter = filter.WithType(objectType) + } + + last, err := strconv.Atoi(c.Query("last", "0")) + if err == nil && last != 0 { + filter = filter.WithLastNDays(last) + } + + dFilter := datefilter.NewDateFilter(c.Query("startDate", ""), c.Query("endDate", "")) + if dFilter.IsStartValid { + filter = filter.WithStartDate(dFilter.Start) + } + + if dFilter.IsEndValid { + filter = filter.WithEndDate(dFilter.End) + } + + selector := c.Query("selector") + if selector != "" { + filter = filter.WithSelector(selector) + } + + return filter +} diff --git a/internal/app/api/v1/executions_test.go b/internal/app/api/deprecatedv1/executions_test.go similarity index 91% rename from internal/app/api/v1/executions_test.go rename to internal/app/api/deprecatedv1/executions_test.go index f7e5646be75..1d016126308 100644 --- a/internal/app/api/v1/executions_test.go +++ b/internal/app/api/deprecatedv1/executions_test.go @@ -1,4 +1,4 @@ -package v1 +package deprecatedv1 import ( "context" @@ -7,9 +7,6 @@ import ( "testing" "time" - "github.com/kubeshop/testkube/pkg/featureflags" - "github.com/kubeshop/testkube/pkg/repository/result" - "github.com/gofiber/fiber/v2" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -18,6 +15,10 @@ import ( k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/kubeshop/testkube/cmd/api-server/commons" + "github.com/kubeshop/testkube/pkg/featureflags" + "github.com/kubeshop/testkube/pkg/repository/result" + executorv1 "github.com/kubeshop/testkube-operator/api/executor/v1" executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" "github.com/kubeshop/testkube/pkg/api/v1/testkube" @@ -26,21 +27,27 @@ import ( "github.com/kubeshop/testkube/pkg/log" logclient "github.com/kubeshop/testkube/pkg/logs/client" "github.com/kubeshop/testkube/pkg/logs/events" - "github.com/kubeshop/testkube/pkg/server" ) -func TestTestkubeAPI_ExecutionLogsHandler(t *testing.T) { +func TestDeprecatedTestkubeAPI_ExecutionLogsHandler(t *testing.T) { app := fiber.New() resultRepo := MockExecutionResultsRepository{} executor := &MockExecutor{} - s := &TestkubeAPI{ - HTTPServer: server.HTTPServer{ - Mux: app, - Log: log.DefaultLogger, - }, - ExecutionResults: &resultRepo, - ExecutorsClient: getMockExecutorClient(), - Executor: executor, + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockDeprecatedRepositories := commons.NewMockDeprecatedRepositories(mockCtrl) + mockDeprecatedRepositories.EXPECT().TestResults().Return(&resultRepo).AnyTimes() + + mockDeprecatedClients := commons.NewMockDeprecatedClients(mockCtrl) + mockDeprecatedClients.EXPECT().Executors().Return(getMockExecutorClient()).AnyTimes() + + s := &DeprecatedTestkubeAPI{ + Log: log.DefaultLogger, + DeprecatedRepositories: mockDeprecatedRepositories, + DeprecatedClients: mockDeprecatedClients, + Executor: executor, } app.Get("/executions/:executionID/logs", s.ExecutionLogsHandler()) @@ -124,7 +131,7 @@ func TestTestkubeAPI_ExecutionLogsHandler(t *testing.T) { } } -func TestTestkubeAPI_ExecutionLogsHandlerV2(t *testing.T) { +func TestDeprecatedTestkubeAPI_ExecutionLogsHandlerV2(t *testing.T) { app := fiber.New() mockCtrl := gomock.NewController(t) @@ -148,11 +155,8 @@ func TestTestkubeAPI_ExecutionLogsHandlerV2(t *testing.T) { }() grpcClient.EXPECT().Get(gomock.Any(), "test-execution-1").Return(out, nil) - s := &TestkubeAPI{ - HTTPServer: server.HTTPServer{ - Mux: app, - Log: log.DefaultLogger, - }, + s := &DeprecatedTestkubeAPI{ + Log: log.DefaultLogger, featureFlags: featureflags.FeatureFlags{LogsV2: true}, logGrpcClient: grpcClient, } diff --git a/internal/app/api/v1/executor.go b/internal/app/api/deprecatedv1/executor.go similarity index 82% rename from internal/app/api/v1/executor.go rename to internal/app/api/deprecatedv1/executor.go index a5d601ccf83..a720208d321 100644 --- a/internal/app/api/v1/executor.go +++ b/internal/app/api/deprecatedv1/executor.go @@ -1,4 +1,4 @@ -package v1 +package deprecatedv1 import ( "bytes" @@ -10,12 +10,13 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" executorv1 "github.com/kubeshop/testkube-operator/api/executor/v1" + "github.com/kubeshop/testkube/internal/app/api/apiutils" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/crd" executorsmapper "github.com/kubeshop/testkube/pkg/mapper/executors" ) -func (s TestkubeAPI) CreateExecutorHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) CreateExecutorHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to create executor" var executor executorv1.Executor @@ -35,14 +36,14 @@ func (s TestkubeAPI) CreateExecutorHandler() fiber.Handler { if c.Accepts(mediaTypeJSON, mediaTypeYAML) == mediaTypeYAML { request.QuoteExecutorTextFields() data, err := crd.GenerateYAML(crd.TemplateExecutor, []testkube.ExecutorUpsertRequest{request}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } executor = executorsmapper.MapAPIToCRD(request) executor.Namespace = s.Namespace } - created, err := s.ExecutorsClient.Create(&executor) + created, err := s.DeprecatedClients.Executors().Create(&executor) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not create executor: %w", errPrefix, err)) } @@ -58,7 +59,7 @@ func (s TestkubeAPI) CreateExecutorHandler() fiber.Handler { } } -func (s TestkubeAPI) UpdateExecutorHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) UpdateExecutorHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to update executor" var request testkube.ExecutorUpdateRequest @@ -84,7 +85,7 @@ func (s TestkubeAPI) UpdateExecutorHandler() fiber.Handler { } errPrefix = errPrefix + " " + name // we need to get resource first and load its metadata.ResourceVersion - executor, err := s.ExecutorsClient.Get(name) + executor, err := s.DeprecatedClients.Executors().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: client found no executor: %w", errPrefix, err)) @@ -96,7 +97,7 @@ func (s TestkubeAPI) UpdateExecutorHandler() fiber.Handler { // map update executor but load spec only to not override metadata.ResourceVersion executorSpec := executorsmapper.MapUpdateToSpec(request, executor) - updatedExecutor, err := s.ExecutorsClient.Update(executorSpec) + updatedExecutor, err := s.DeprecatedClients.Executors().Update(executorSpec) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not update executor: %w", errPrefix, err)) } @@ -111,10 +112,10 @@ func (s TestkubeAPI) UpdateExecutorHandler() fiber.Handler { } } -func (s TestkubeAPI) ListExecutorsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ListExecutorsHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to list executors" - list, err := s.ExecutorsClient.List(c.Query("selector")) + list, err := s.DeprecatedClients.Executors().List(c.Query("selector")) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not list executors: %w", errPrefix, err)) } @@ -128,7 +129,7 @@ func (s TestkubeAPI) ListExecutorsHandler() fiber.Handler { } data, err := crd.GenerateYAML(crd.TemplateExecutor, results) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } results := []testkube.ExecutorDetails{} @@ -140,12 +141,12 @@ func (s TestkubeAPI) ListExecutorsHandler() fiber.Handler { } } -func (s TestkubeAPI) GetExecutorHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) GetExecutorHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("name") errPrefix := fmt.Sprintf("failed to get executor %s", name) - item, err := s.ExecutorsClient.Get(name) + item, err := s.DeprecatedClients.Executors().Get(name) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not get executor: %w", errPrefix, err)) } @@ -154,7 +155,7 @@ func (s TestkubeAPI) GetExecutorHandler() fiber.Handler { result := executorsmapper.MapCRDToAPI(*item) result.QuoteExecutorTextFields() data, err := crd.GenerateYAML(crd.TemplateExecutor, []testkube.ExecutorUpsertRequest{result}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } result := executorsmapper.MapExecutorCRDToExecutorDetails(*item) @@ -162,12 +163,12 @@ func (s TestkubeAPI) GetExecutorHandler() fiber.Handler { } } -func (s TestkubeAPI) DeleteExecutorHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) DeleteExecutorHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("name") errPrefix := fmt.Sprintf("failed to delete executor %s", name) - err := s.ExecutorsClient.Delete(name) + err := s.DeprecatedClients.Executors().Delete(name) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not delete executor: %w", errPrefix, err)) } @@ -184,10 +185,10 @@ func (s TestkubeAPI) DeleteExecutorHandler() fiber.Handler { } } -func (s TestkubeAPI) DeleteExecutorsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) DeleteExecutorsHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to delete executors" - err := s.ExecutorsClient.DeleteByLabels(c.Query("selector")) + err := s.DeprecatedClients.Executors().DeleteByLabels(c.Query("selector")) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not delete executors: %w", errPrefix, err)) } @@ -197,7 +198,7 @@ func (s TestkubeAPI) DeleteExecutorsHandler() fiber.Handler { } } -func (s TestkubeAPI) GetExecutorByTestTypeHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) GetExecutorByTestTypeHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to get executor by test type" @@ -206,7 +207,7 @@ func (s TestkubeAPI) GetExecutorByTestTypeHandler() fiber.Handler { return s.Error(c, http.StatusBadRequest, fmt.Errorf("%s: could not fine test type", errPrefix)) } - item, err := s.ExecutorsClient.GetByType(testType) + item, err := s.DeprecatedClients.Executors().GetByType(testType) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not get executor: %w", errPrefix, err)) } @@ -215,7 +216,7 @@ func (s TestkubeAPI) GetExecutorByTestTypeHandler() fiber.Handler { result := executorsmapper.MapCRDToAPI(*item) result.QuoteExecutorTextFields() data, err := crd.GenerateYAML(crd.TemplateExecutor, []testkube.ExecutorUpsertRequest{result}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } result := executorsmapper.MapExecutorCRDToExecutorDetails(*item) diff --git a/internal/app/api/v1/graphql.go b/internal/app/api/deprecatedv1/graphql.go similarity index 60% rename from internal/app/api/v1/graphql.go rename to internal/app/api/deprecatedv1/graphql.go index 1f1b5b7ad13..7a95bcb706d 100644 --- a/internal/app/api/v1/graphql.go +++ b/internal/app/api/deprecatedv1/graphql.go @@ -1,7 +1,8 @@ -package v1 +package deprecatedv1 import ( "context" + "fmt" "net" "net/http" @@ -10,16 +11,16 @@ import ( ) // RunGraphQLServer runs GraphQL server on go net/http server -func (s *TestkubeAPI) RunGraphQLServer(ctx context.Context, port string) error { - srv := graphql.GetServer(s.Events.Bus, s.ExecutorsClient) +func (s *DeprecatedTestkubeAPI) RunGraphQLServer(ctx context.Context) error { + srv := graphql.GetServer(s.Events.Bus, s.DeprecatedClients.Executors()) mux := http.NewServeMux() mux.Handle("/graphql", srv) httpSrv := &http.Server{Handler: mux} - log.DefaultLogger.Infow("running GraphQL server", "port", port) + log.DefaultLogger.Infow("running GraphQL server", "port", s.graphqlPort) - l, err := net.Listen("tcp", ":"+port) + l, err := net.Listen("tcp", fmt.Sprintf(":%d", s.graphqlPort)) if err != nil { return err } diff --git a/internal/app/api/deprecatedv1/handlers.go b/internal/app/api/deprecatedv1/handlers.go new file mode 100644 index 00000000000..908cea8b735 --- /dev/null +++ b/internal/app/api/deprecatedv1/handlers.go @@ -0,0 +1,55 @@ +package deprecatedv1 + +import ( + "fmt" + "net/http" + + "github.com/gofiber/fiber/v2" + "github.com/pkg/errors" + + "github.com/kubeshop/testkube/internal/app/api/apiutils" +) + +const ( + // mediaTypeJSON is json media type + mediaTypeJSON = "application/json" + // mediaTypeYAML is yaml media type + mediaTypeYAML = "text/yaml" +) + +// Warn writes RFC-7807 json problem to response +func (s *DeprecatedTestkubeAPI) Warn(c *fiber.Ctx, status int, err error, context ...interface{}) error { + return apiutils.SendWarn(c, status, err, context...) +} + +// Error writes RFC-7807 json problem to response +func (s *DeprecatedTestkubeAPI) Error(c *fiber.Ctx, status int, err error, context ...interface{}) error { + return apiutils.SendError(c, status, err, context...) +} + +func (s *DeprecatedTestkubeAPI) NotImplemented(c *fiber.Ctx) error { + return s.Error(c, http.StatusNotImplemented, errors.New("not implemented yet")) +} + +func (s *DeprecatedTestkubeAPI) BadGateway(c *fiber.Ctx, prefix, description string, err error) error { + return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: %s: %w", prefix, description, err)) +} + +func (s *DeprecatedTestkubeAPI) InternalError(c *fiber.Ctx, prefix, description string, err error) error { + return s.Error(c, http.StatusInternalServerError, fmt.Errorf("%s: %s: %w", prefix, description, err)) +} + +func (s *DeprecatedTestkubeAPI) BadRequest(c *fiber.Ctx, prefix, description string, err error) error { + return s.Error(c, http.StatusBadRequest, fmt.Errorf("%s: %s: %w", prefix, description, err)) +} + +func (s *DeprecatedTestkubeAPI) NotFound(c *fiber.Ctx, prefix, description string, err error) error { + return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: %s: %w", prefix, description, err)) +} + +func (s *DeprecatedTestkubeAPI) ClientError(c *fiber.Ctx, prefix string, err error) error { + if apiutils.IsNotFound(err) { + return s.NotFound(c, prefix, "client not found", err) + } + return s.BadGateway(c, prefix, "client problem", err) +} diff --git a/internal/app/api/deprecatedv1/server.go b/internal/app/api/deprecatedv1/server.go new file mode 100644 index 00000000000..0a2d1e8ec78 --- /dev/null +++ b/internal/app/api/deprecatedv1/server.go @@ -0,0 +1,288 @@ +package deprecatedv1 + +import ( + "fmt" + "io" + "net" + "reflect" + "sync" + "syscall" + + "github.com/pkg/errors" + "go.uber.org/zap" + + "github.com/gofiber/fiber/v2" + + "github.com/kubeshop/testkube/cmd/api-server/commons" + "github.com/kubeshop/testkube/internal/config" + "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/log" + + "github.com/kubeshop/testkube/internal/app/api/metrics" + "github.com/kubeshop/testkube/pkg/event" + "github.com/kubeshop/testkube/pkg/event/bus" + "github.com/kubeshop/testkube/pkg/executor/client" + "github.com/kubeshop/testkube/pkg/featureflags" + logsclient "github.com/kubeshop/testkube/pkg/logs/client" + "github.com/kubeshop/testkube/pkg/scheduler" + "github.com/kubeshop/testkube/pkg/secret" + "github.com/kubeshop/testkube/pkg/server" + "github.com/kubeshop/testkube/pkg/storage" +) + +func NewDeprecatedTestkubeAPI( + deprecatedRepositories commons.DeprecatedRepositories, + deprecatedClients commons.DeprecatedClients, + namespace string, + secretClient secret.Interface, + eventsEmitter *event.Emitter, + executor client.Executor, + containerExecutor client.Executor, + metrics metrics.Metrics, + scheduler *scheduler.Scheduler, + graphqlPort int, + artifactsStorage storage.ArtifactsStorage, + mode string, + eventsBus bus.Bus, + secretConfig testkube.SecretConfig, + ff featureflags.FeatureFlags, + logsStream logsclient.Stream, + logGrpcClient logsclient.StreamGetter, + proContext *config.ProContext, + storageParams StorageParams, +) DeprecatedTestkubeAPI { + return DeprecatedTestkubeAPI{ + Log: log.DefaultLogger, + DeprecatedRepositories: deprecatedRepositories, + DeprecatedClients: deprecatedClients, + SecretClient: secretClient, + Metrics: metrics, + Events: eventsEmitter, + Namespace: namespace, + Executor: executor, + ContainerExecutor: containerExecutor, + storageParams: storageParams, + scheduler: scheduler, + graphqlPort: graphqlPort, + ArtifactsStorage: artifactsStorage, + mode: mode, + eventsBus: eventsBus, + secretConfig: secretConfig, + featureFlags: ff, + logsStream: logsStream, + logGrpcClient: logGrpcClient, + proContext: proContext, + } +} + +type DeprecatedTestkubeAPI struct { + Log *zap.SugaredLogger + Executor client.Executor + ContainerExecutor client.Executor + DeprecatedRepositories commons.DeprecatedRepositories + DeprecatedClients commons.DeprecatedClients + SecretClient secret.Interface + Metrics metrics.Metrics + storageParams StorageParams + Namespace string + Events *event.Emitter + scheduler *scheduler.Scheduler + graphqlPort int + ArtifactsStorage storage.ArtifactsStorage + mode string + eventsBus bus.Bus + secretConfig testkube.SecretConfig + featureFlags featureflags.FeatureFlags + logsStream logsclient.Stream + logGrpcClient logsclient.StreamGetter + proContext *config.ProContext +} + +type StorageParams struct { + SSL bool `envconfig:"STORAGE_SSL" default:"false"` + SkipVerify bool `envconfig:"STORAGE_SKIP_VERIFY" default:"false"` + CertFile string `envconfig:"STORAGE_CERT_FILE"` + KeyFile string `envconfig:"STORAGE_KEY_FILE"` + CAFile string `envconfig:"STORAGE_CA_FILE"` + Endpoint string + AccessKeyId string + SecretAccessKey string + Region string + Token string + Bucket string +} + +func (s *DeprecatedTestkubeAPI) Init(server server.HTTPServer) { + root := server.Routes + + executors := root.Group("/executors") + + executors.Post("/", s.CreateExecutorHandler()) + executors.Get("/", s.ListExecutorsHandler()) + executors.Get("/:name", s.GetExecutorHandler()) + executors.Patch("/:name", s.UpdateExecutorHandler()) + executors.Delete("/:name", s.DeleteExecutorHandler()) + executors.Delete("/", s.DeleteExecutorsHandler()) + + executorByTypes := root.Group("/executor-by-types") + executorByTypes.Get("/", s.GetExecutorByTestTypeHandler()) + + executions := root.Group("/executions") + + executions.Get("/", s.ListExecutionsHandler()) + executions.Post("/", s.ExecuteTestsHandler()) + executions.Get("/:executionID", s.GetExecutionHandler()) + executions.Get("/:executionID/artifacts", s.ListArtifactsHandler()) + executions.Get("/:executionID/logs", s.ExecutionLogsHandler()) + executions.Get("/:executionID/logs/stream", s.ExecutionLogsStreamHandler()) + executions.Get("/:executionID/logs/v2", s.ExecutionLogsHandlerV2()) + executions.Get("/:executionID/logs/stream/v2", s.ExecutionLogsStreamHandlerV2()) + executions.Get("/:executionID/artifacts/:filename", s.GetArtifactHandler()) + executions.Get("/:executionID/artifact-archive", s.GetArtifactArchiveHandler()) + + tests := root.Group("/tests") + + tests.Get("/", s.ListTestsHandler()) + tests.Post("/", s.CreateTestHandler()) + tests.Patch("/:id", s.UpdateTestHandler()) + tests.Delete("/", s.DeleteTestsHandler()) + + tests.Get("/:id", s.GetTestHandler()) + tests.Delete("/:id", s.DeleteTestHandler()) + tests.Post("/:id/abort", s.AbortTestHandler()) + + tests.Get("/:id/metrics", s.TestMetricsHandler()) + + tests.Post("/:id/executions", s.ExecuteTestsHandler()) + + tests.Get("/:id/executions", s.ListExecutionsHandler()) + tests.Get("/:id/executions/:executionID", s.GetExecutionHandler()) + tests.Patch("/:id/executions/:executionID", s.AbortExecutionHandler()) + + testWithExecutions := server.Routes.Group("/test-with-executions") + testWithExecutions.Get("/", s.ListTestWithExecutionsHandler()) + testWithExecutions.Get("/:id", s.GetTestWithExecutionHandler()) + + testsuites := root.Group("/test-suites") + + testsuites.Post("/", s.CreateTestSuiteHandler()) + testsuites.Patch("/:id", s.UpdateTestSuiteHandler()) + testsuites.Get("/", s.ListTestSuitesHandler()) + testsuites.Delete("/", s.DeleteTestSuitesHandler()) + testsuites.Get("/:id", s.GetTestSuiteHandler()) + testsuites.Delete("/:id", s.DeleteTestSuiteHandler()) + testsuites.Post("/:id/abort", s.AbortTestSuiteHandler()) + + testsuites.Post("/:id/executions", s.ExecuteTestSuitesHandler()) + testsuites.Get("/:id/executions", s.ListTestSuiteExecutionsHandler()) + testsuites.Get("/:id/executions/:executionID", s.GetTestSuiteExecutionHandler()) + testsuites.Get("/:id/executions/:executionID/artifacts", s.ListTestSuiteArtifactsHandler()) + testsuites.Patch("/:id/executions/:executionID", s.AbortTestSuiteExecutionHandler()) + + testsuites.Get("/:id/tests", s.ListTestSuiteTestsHandler()) + + testsuites.Get("/:id/metrics", s.TestSuiteMetricsHandler()) + + testSuiteExecutions := root.Group("/test-suite-executions") + testSuiteExecutions.Get("/", s.ListTestSuiteExecutionsHandler()) + testSuiteExecutions.Post("/", s.ExecuteTestSuitesHandler()) + testSuiteExecutions.Get("/:executionID", s.GetTestSuiteExecutionHandler()) + testSuiteExecutions.Get("/:executionID/artifacts", s.ListTestSuiteArtifactsHandler()) + testSuiteExecutions.Patch("/:executionID", s.AbortTestSuiteExecutionHandler()) + + testSuiteWithExecutions := root.Group("/test-suite-with-executions") + testSuiteWithExecutions.Get("/", s.ListTestSuiteWithExecutionsHandler()) + testSuiteWithExecutions.Get("/:id", s.GetTestSuiteWithExecutionHandler()) + + testsources := root.Group("/test-sources") + testsources.Post("/", s.CreateTestSourceHandler()) + testsources.Get("/", s.ListTestSourcesHandler()) + testsources.Patch("/", s.ProcessTestSourceBatchHandler()) + testsources.Get("/:name", s.GetTestSourceHandler()) + testsources.Patch("/:name", s.UpdateTestSourceHandler()) + testsources.Delete("/:name", s.DeleteTestSourceHandler()) + testsources.Delete("/", s.DeleteTestSourcesHandler()) + + templates := root.Group("/templates") + + templates.Post("/", s.CreateTemplateHandler()) + templates.Patch("/:name", s.UpdateTemplateHandler()) + templates.Get("/", s.ListTemplatesHandler()) + templates.Get("/:name", s.GetTemplateHandler()) + templates.Delete("/:name", s.DeleteTemplateHandler()) + templates.Delete("/", s.DeleteTemplatesHandler()) + + slack := root.Group("/slack") + slack.Get("/", s.OauthHandler()) + + files := root.Group("/uploads") + files.Post("/", s.UploadFiles()) + + // set up proxy for the internal GraphQL server + server.Mux.All("/graphql", func(c *fiber.Ctx) error { + // Connect to server + serverConn, err := net.Dial("tcp", fmt.Sprintf(":%d", s.graphqlPort)) + if err != nil { + s.Log.Errorw("could not connect to GraphQL server as a proxy", "error", err) + return err + } + + // Resend headers to the server + _, err = serverConn.Write(c.Request().Header.Header()) + if err != nil { + serverConn.Close() + s.Log.Errorw("error while sending headers to GraphQL server", "error", err) + return err + } + + // Resend body to the server + _, err = serverConn.Write(c.Body()) + if err != nil && err != io.EOF { + serverConn.Close() + s.Log.Errorw("error while reading GraphQL client data", "error", err) + return err + } + + // Handle optional WebSocket connection + c.Context().HijackSetNoResponse(true) + c.Context().Hijack(func(clientConn net.Conn) { + // Close the connection afterward + defer serverConn.Close() + defer clientConn.Close() + + // Extract Unix connection + serverSock, ok := serverConn.(*net.TCPConn) + if !ok { + s.Log.Errorw("error while building TCPConn out ouf serverConn", "error", err) + return + } + clientSock, ok := reflect.Indirect(reflect.ValueOf(clientConn)).FieldByName("Conn").Interface().(*net.TCPConn) + if !ok { + s.Log.Errorw("error while building TCPConn out of hijacked connection", "error", err) + return + } + + // Duplex communication between client and GraphQL server + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + _, err := io.Copy(clientSock, serverSock) + if err != nil && err != io.EOF && !errors.Is(err, syscall.ECONNRESET) && !errors.Is(err, syscall.EPIPE) { + s.Log.Errorw("error while reading GraphQL client data", "error", err) + } + serverSock.CloseWrite() + }() + go func() { + defer wg.Done() + _, err = io.Copy(serverSock, clientSock) + if err != nil && err != io.EOF { + s.Log.Errorw("error while reading GraphQL server data", "error", err) + } + clientSock.CloseWrite() + }() + wg.Wait() + }) + return nil + }) +} diff --git a/internal/app/api/v1/slack_oauth.go b/internal/app/api/deprecatedv1/slack_oauth.go similarity index 96% rename from internal/app/api/v1/slack_oauth.go rename to internal/app/api/deprecatedv1/slack_oauth.go index 48758fc681a..00d375bd347 100644 --- a/internal/app/api/v1/slack_oauth.go +++ b/internal/app/api/deprecatedv1/slack_oauth.go @@ -1,4 +1,4 @@ -package v1 +package deprecatedv1 import ( "encoding/json" @@ -37,7 +37,7 @@ type oauthResponse struct { } // OauthHandler creates a handler for slack authentication -func (s TestkubeAPI) OauthHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) OauthHandler() fiber.Handler { return func(c *fiber.Ctx) error { errStr := c.Query("error", "") diff --git a/internal/app/api/v1/template.go b/internal/app/api/deprecatedv1/template.go similarity index 82% rename from internal/app/api/v1/template.go rename to internal/app/api/deprecatedv1/template.go index dc4c026b995..a34fc2b049e 100644 --- a/internal/app/api/v1/template.go +++ b/internal/app/api/deprecatedv1/template.go @@ -1,4 +1,4 @@ -package v1 +package deprecatedv1 import ( "bytes" @@ -10,12 +10,13 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" templatev1 "github.com/kubeshop/testkube-operator/api/template/v1" + "github.com/kubeshop/testkube/internal/app/api/apiutils" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/crd" templatesmapper "github.com/kubeshop/testkube/pkg/mapper/templates" ) -func (s TestkubeAPI) CreateTemplateHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) CreateTemplateHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to create template" var template templatev1.Template @@ -38,14 +39,14 @@ func (s TestkubeAPI) CreateTemplateHandler() fiber.Handler { } data, err := crd.GenerateYAML(crd.TemplateTemplate, []testkube.TemplateCreateRequest{request}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } template = templatesmapper.MapAPIToCRD(request) template.Namespace = s.Namespace } - created, err := s.TemplatesClient.Create(&template) + created, err := s.DeprecatedClients.Templates().Create(&template) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not create template: %w", errPrefix, err)) } @@ -55,7 +56,7 @@ func (s TestkubeAPI) CreateTemplateHandler() fiber.Handler { } } -func (s TestkubeAPI) UpdateTemplateHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) UpdateTemplateHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to update template" var request testkube.TemplateUpdateRequest @@ -81,7 +82,7 @@ func (s TestkubeAPI) UpdateTemplateHandler() fiber.Handler { } errPrefix = errPrefix + " " + name // we need to get resource first and load its metadata.ResourceVersion - template, err := s.TemplatesClient.Get(name) + template, err := s.DeprecatedClients.Templates().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: client found no template: %w", errPrefix, err)) @@ -93,7 +94,7 @@ func (s TestkubeAPI) UpdateTemplateHandler() fiber.Handler { // map update template but load spec only to not override metadata.ResourceVersion templateSpec := templatesmapper.MapUpdateToSpec(request, template) - updatedTemplate, err := s.TemplatesClient.Update(templateSpec) + updatedTemplate, err := s.DeprecatedClients.Templates().Update(templateSpec) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not update template: %w", errPrefix, err)) } @@ -102,11 +103,11 @@ func (s TestkubeAPI) UpdateTemplateHandler() fiber.Handler { } } -func (s TestkubeAPI) ListTemplatesHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ListTemplatesHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to list templates" - list, err := s.TemplatesClient.List(c.Query("selector")) + list, err := s.DeprecatedClients.Templates().List(c.Query("selector")) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not list templates: %w", errPrefix, err)) } @@ -125,19 +126,19 @@ func (s TestkubeAPI) ListTemplatesHandler() fiber.Handler { } data, err := crd.GenerateYAML(crd.TemplateTemplate, results) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } return c.JSON(results) } } -func (s TestkubeAPI) GetTemplateHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) GetTemplateHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("name") errPrefix := fmt.Sprintf("failed to get template %s", name) - item, err := s.TemplatesClient.Get(name) + item, err := s.DeprecatedClients.Templates().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: template not found: %w", errPrefix, err)) @@ -152,19 +153,19 @@ func (s TestkubeAPI) GetTemplateHandler() fiber.Handler { } data, err := crd.GenerateYAML(crd.TemplateTemplate, []testkube.Template{result}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } return c.JSON(result) } } -func (s TestkubeAPI) DeleteTemplateHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) DeleteTemplateHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("name") errPrefix := fmt.Sprintf("failed to delete template %s", name) - err := s.TemplatesClient.Delete(name) + err := s.DeprecatedClients.Templates().Delete(name) if err != nil { if errors.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: template not found: %w", errPrefix, err)) @@ -177,11 +178,11 @@ func (s TestkubeAPI) DeleteTemplateHandler() fiber.Handler { } } -func (s TestkubeAPI) DeleteTemplatesHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) DeleteTemplatesHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to delete templates" - err := s.TemplatesClient.DeleteByLabels(c.Query("selector")) + err := s.DeprecatedClients.Templates().DeleteByLabels(c.Query("selector")) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not delete templates: %w", errPrefix, err)) } diff --git a/internal/app/api/v1/tests.go b/internal/app/api/deprecatedv1/tests.go similarity index 87% rename from internal/app/api/v1/tests.go rename to internal/app/api/deprecatedv1/tests.go index 77082906c73..88ae673cfc0 100644 --- a/internal/app/api/v1/tests.go +++ b/internal/app/api/deprecatedv1/tests.go @@ -1,4 +1,4 @@ -package v1 +package deprecatedv1 import ( "bytes" @@ -18,6 +18,7 @@ import ( "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" testsclientv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" "github.com/kubeshop/testkube-operator/pkg/secret" + "github.com/kubeshop/testkube/internal/app/api/apiutils" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/crd" "github.com/kubeshop/testkube/pkg/executor/client" @@ -27,14 +28,14 @@ import ( ) // GetTestHandler is method for getting an existing test -func (s TestkubeAPI) GetTestHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) GetTestHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("id") if name == "" { return s.Error(c, http.StatusBadRequest, fmt.Errorf("failed to get test: id cannot be empty")) } errPrefix := fmt.Sprintf("failed to get test %s", name) - crTest, err := s.TestsClient.Get(name) + crTest, err := s.DeprecatedClients.Tests().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: client was unable to find test: %w", errPrefix, err)) @@ -50,7 +51,7 @@ func (s TestkubeAPI) GetTestHandler() fiber.Handler { if err != nil { return s.Error(c, http.StatusBadRequest, fmt.Errorf("%s: could not build CRD: %w", errPrefix, err)) } - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } return c.JSON(test) @@ -58,14 +59,14 @@ func (s TestkubeAPI) GetTestHandler() fiber.Handler { } // GetTestWithExecutionHandler is method for getting an existing test with execution -func (s TestkubeAPI) GetTestWithExecutionHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) GetTestWithExecutionHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("id") if name == "" { return s.Error(c, http.StatusBadRequest, fmt.Errorf("failed to get test with execution: id cannot be empty")) } errPrefix := fmt.Sprintf("failed to get test %s with execution", name) - crTest, err := s.TestsClient.Get(name) + crTest, err := s.DeprecatedClients.Tests().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: client failed to find test: %w", errPrefix, err)) @@ -81,11 +82,11 @@ func (s TestkubeAPI) GetTestWithExecutionHandler() fiber.Handler { if err != nil { return s.Error(c, http.StatusBadRequest, fmt.Errorf("%s: could not build CRD: %w", errPrefix, err)) } - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } ctx := c.Context() - execution, err := s.ExecutionResults.GetLatestByTest(ctx, name) + execution, err := s.DeprecatedRepositories.TestResults().GetLatestByTest(ctx, name) if err != nil && err != mongo.ErrNoDocuments { return s.Error(c, http.StatusInternalServerError, fmt.Errorf("%s: failed to get execution: %w", errPrefix, err)) } @@ -97,9 +98,9 @@ func (s TestkubeAPI) GetTestWithExecutionHandler() fiber.Handler { } } -func (s TestkubeAPI) getFilteredTestList(c *fiber.Ctx) (*testsv3.TestList, error) { +func (s *DeprecatedTestkubeAPI) getFilteredTestList(c *fiber.Ctx) (*testsv3.TestList, error) { - crTests, err := s.TestsClient.List(c.Query("selector")) + crTests, err := s.DeprecatedClients.Tests().List(c.Query("selector")) if err != nil { return nil, fmt.Errorf("client failed to list tests: %w", err) } @@ -128,7 +129,7 @@ func (s TestkubeAPI) getFilteredTestList(c *fiber.Ctx) (*testsv3.TestList, error } // ListTestsHandler is a method for getting list of all available tests -func (s TestkubeAPI) ListTestsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ListTestsHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to list tests" crTests, err := s.getFilteredTestList(c) @@ -146,7 +147,7 @@ func (s TestkubeAPI) ListTestsHandler() fiber.Handler { if err != nil { return s.Error(c, http.StatusBadRequest, fmt.Errorf("%s: could not build CRD: %w", errPrefix, err)) } - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } return c.JSON(tests) @@ -154,7 +155,7 @@ func (s TestkubeAPI) ListTestsHandler() fiber.Handler { } // ListTestsHandler is a method for getting list of all available tests -func (s TestkubeAPI) TestMetricsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) TestMetricsHandler() fiber.Handler { return func(c *fiber.Ctx) error { testName := c.Params("id") @@ -170,7 +171,7 @@ func (s TestkubeAPI) TestMetricsHandler() fiber.Handler { last = DefaultLastDays } - metrics, err := s.ExecutionResults.GetTestMetrics(context.Background(), testName, limit, last) + metrics, err := s.DeprecatedRepositories.TestResults().GetTestMetrics(context.Background(), testName, limit, last) if err != nil { return s.Error(c, http.StatusInternalServerError, fmt.Errorf("failed to get metrics for test %s: %w", testName, err)) } @@ -180,8 +181,8 @@ func (s TestkubeAPI) TestMetricsHandler() fiber.Handler { } // getLatestExecutions return latest executions either by starttime or endtime for tests -func (s TestkubeAPI) getLatestExecutions(ctx context.Context, testNames []string) (map[string]testkube.Execution, error) { - executions, err := s.ExecutionResults.GetLatestByTests(ctx, testNames) +func (s *DeprecatedTestkubeAPI) getLatestExecutions(ctx context.Context, testNames []string) (map[string]testkube.Execution, error) { + executions, err := s.DeprecatedRepositories.TestResults().GetLatestByTests(ctx, testNames) if err != nil && err != mongo.ErrNoDocuments { return nil, fmt.Errorf("could not get latest executions for tests %s sorted by start time: %w", testNames, err) } @@ -194,7 +195,7 @@ func (s TestkubeAPI) getLatestExecutions(ctx context.Context, testNames []string } // ListTestWithExecutionsHandler is a method for getting list of all available test with latest executions -func (s TestkubeAPI) ListTestWithExecutionsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ListTestWithExecutionsHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to list tests with executions" crTests, err := s.getFilteredTestList(c) @@ -212,7 +213,7 @@ func (s TestkubeAPI) ListTestWithExecutionsHandler() fiber.Handler { if err != nil { return s.Error(c, http.StatusBadRequest, fmt.Errorf("%s: could not build CRD: %w", errPrefix, err)) } - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } ctx := c.Context() @@ -315,7 +316,7 @@ func (s TestkubeAPI) ListTestWithExecutionsHandler() fiber.Handler { } // CreateTestHandler creates new test CR based on test content -func (s TestkubeAPI) CreateTestHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) CreateTestHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to create test" var test *testsv3.Test @@ -349,7 +350,7 @@ func (s TestkubeAPI) CreateTestHandler() fiber.Handler { return s.Error(c, http.StatusBadRequest, fmt.Errorf("%s: could not build CRD: %w", errPrefix, err)) } - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } s.Log.Infow("creating test", "request", request) @@ -361,7 +362,7 @@ func (s TestkubeAPI) CreateTestHandler() fiber.Handler { } } - createdTest, err := s.TestsClient.Create(test, !s.secretConfig.AutoCreate, tests.Option{Secrets: secrets}) + createdTest, err := s.DeprecatedClients.Tests().Create(test, !s.secretConfig.AutoCreate, tests.Option{Secrets: secrets}) s.Metrics.IncCreateTest(test.Spec.Type_, err) @@ -375,7 +376,7 @@ func (s TestkubeAPI) CreateTestHandler() fiber.Handler { } // UpdateTestHandler updates an existing test CR based on test content -func (s TestkubeAPI) UpdateTestHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) UpdateTestHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to update test" var request testkube.TestUpdateRequest @@ -407,7 +408,7 @@ func (s TestkubeAPI) UpdateTestHandler() fiber.Handler { } // we need to get resource first and load its metadata.ResourceVersion - test, err := s.TestsClient.Get(name) + test, err := s.DeprecatedClients.Tests().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: client could not find test: %w", errPrefix, err)) @@ -441,9 +442,9 @@ func (s TestkubeAPI) UpdateTestHandler() fiber.Handler { var updatedTest *testsv3.Test if option != nil { - updatedTest, err = s.TestsClient.Update(testSpec, !s.secretConfig.AutoCreate, *option) + updatedTest, err = s.DeprecatedClients.Tests().Update(testSpec, !s.secretConfig.AutoCreate, *option) } else { - updatedTest, err = s.TestsClient.Update(testSpec, !s.secretConfig.AutoCreate) + updatedTest, err = s.DeprecatedClients.Tests().Update(testSpec, !s.secretConfig.AutoCreate) } s.Metrics.IncUpdateTest(testSpec.Spec.Type_, err) @@ -468,7 +469,7 @@ func isRepositoryEmpty(s testsv3.TestSpec) bool { } // DeleteTestHandler is a method for deleting a test with id -func (s TestkubeAPI) DeleteTestHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) DeleteTestHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("id") if name == "" { @@ -477,7 +478,7 @@ func (s TestkubeAPI) DeleteTestHandler() fiber.Handler { errPrefix := fmt.Sprintf("failed to delete test %s", name) skipCRD := c.Query("skipDeleteCRD", "") if skipCRD != "true" { - err := s.TestsClient.Delete(name) + err := s.DeprecatedClients.Tests().Delete(name) if err != nil { if errors.IsNotFound(err) { return s.Warn(c, http.StatusNotFound, fmt.Errorf("%s: client could not find test: %w", errPrefix, err)) @@ -493,7 +494,7 @@ func (s TestkubeAPI) DeleteTestHandler() fiber.Handler { skipExecutions := c.Query("skipDeleteExecutions", "") if skipExecutions != "true" { // delete executions for test - if err := s.ExecutionResults.DeleteByTest(c.Context(), name); err != nil { + if err := s.DeprecatedRepositories.TestResults().DeleteByTest(c.Context(), name); err != nil { return s.Warn(c, http.StatusInternalServerError, fmt.Errorf("test %s was deleted but deleting test executions returned error: %w", name, err)) } } @@ -503,7 +504,7 @@ func (s TestkubeAPI) DeleteTestHandler() fiber.Handler { } // AbortTestHandler is a method for aborting a executions of a test with id -func (s TestkubeAPI) AbortTestHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) AbortTestHandler() fiber.Handler { return func(c *fiber.Ctx) error { ctx := c.Context() name := c.Params("id") @@ -512,7 +513,7 @@ func (s TestkubeAPI) AbortTestHandler() fiber.Handler { } errPrefix := fmt.Sprintf("failed to abort test %s", name) filter := result.NewExecutionsFilter().WithTestName(name).WithStatus(string(testkube.RUNNING_ExecutionStatus)) - executions, err := s.ExecutionResults.GetExecutions(ctx, filter) + executions, err := s.DeprecatedRepositories.TestResults().GetExecutions(ctx, filter) if err != nil { if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: executions with name %s not found", errPrefix, name)) @@ -536,17 +537,17 @@ func (s TestkubeAPI) AbortTestHandler() fiber.Handler { } // DeleteTestsHandler for deleting all tests -func (s TestkubeAPI) DeleteTestsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) DeleteTestsHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to delete tests" var err error var testNames []string selector := c.Query("selector") if selector == "" { - err = s.TestsClient.DeleteAll() + err = s.DeprecatedClients.Tests().DeleteAll() } else { var testList *testsv3.TestList - testList, err = s.TestsClient.List(selector) + testList, err = s.DeprecatedClients.Tests().List(selector) if err != nil { if !errors.IsNotFound(err) { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not list tests: %w", errPrefix, err)) @@ -557,7 +558,7 @@ func (s TestkubeAPI) DeleteTestsHandler() fiber.Handler { } } - err = s.TestsClient.DeleteByLabels(selector) + err = s.DeprecatedClients.Tests().DeleteByLabels(selector) } if err != nil { @@ -570,9 +571,9 @@ func (s TestkubeAPI) DeleteTestsHandler() fiber.Handler { // delete all executions for tests if selector == "" { - err = s.ExecutionResults.DeleteAll(c.Context()) + err = s.DeprecatedRepositories.TestResults().DeleteAll(c.Context()) } else { - err = s.ExecutionResults.DeleteByTests(c.Context(), testNames) + err = s.DeprecatedRepositories.TestResults().DeleteByTests(c.Context(), testNames) } if err != nil { diff --git a/internal/app/api/v1/tests_test.go b/internal/app/api/deprecatedv1/tests_test.go similarity index 72% rename from internal/app/api/v1/tests_test.go rename to internal/app/api/deprecatedv1/tests_test.go index a6c3f9b9721..5627721e207 100644 --- a/internal/app/api/v1/tests_test.go +++ b/internal/app/api/deprecatedv1/tests_test.go @@ -1,13 +1,10 @@ -package v1 +package deprecatedv1 import ( "net/http/httptest" "testing" - testsv3 "github.com/kubeshop/testkube-operator/api/tests/v3" - testsclientv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" - "github.com/kubeshop/testkube/pkg/log" - "github.com/kubeshop/testkube/pkg/server" + "github.com/golang/mock/gomock" "github.com/gofiber/fiber/v2" "github.com/stretchr/testify/assert" @@ -16,17 +13,25 @@ import ( "k8s.io/apimachinery/pkg/runtime" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + + testsv3 "github.com/kubeshop/testkube-operator/api/tests/v3" + testsclientv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" + "github.com/kubeshop/testkube/cmd/api-server/commons" + "github.com/kubeshop/testkube/pkg/log" ) -func TestTestkubeAPI_DeleteTest(t *testing.T) { +func TestDeprecatedTestkubeAPI_DeleteTest(t *testing.T) { app := fiber.New() - s := &TestkubeAPI{ - HTTPServer: server.HTTPServer{ - Mux: app, - Log: log.DefaultLogger, - }, - TestsClient: getMockTestClient(), + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockDeprecatedClients := commons.NewMockDeprecatedClients(mockCtrl) + mockDeprecatedClients.EXPECT().Tests().Return(getMockTestClient()).AnyTimes() + + s := &DeprecatedTestkubeAPI{ + Log: log.DefaultLogger, + DeprecatedClients: mockDeprecatedClients, } app.Delete("/tests/:id", s.DeleteTestHandler()) diff --git a/internal/app/api/v1/testsource.go b/internal/app/api/deprecatedv1/testsource.go similarity index 83% rename from internal/app/api/v1/testsource.go rename to internal/app/api/deprecatedv1/testsource.go index aa2bae28d6b..30418faa17a 100644 --- a/internal/app/api/v1/testsource.go +++ b/internal/app/api/deprecatedv1/testsource.go @@ -1,4 +1,4 @@ -package v1 +package deprecatedv1 import ( "bytes" @@ -12,13 +12,14 @@ import ( testsourcev1 "github.com/kubeshop/testkube-operator/api/testsource/v1" "github.com/kubeshop/testkube-operator/pkg/client/testsources/v1" "github.com/kubeshop/testkube-operator/pkg/secret" + "github.com/kubeshop/testkube/internal/app/api/apiutils" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/crd" "github.com/kubeshop/testkube/pkg/executor/client" testsourcesmapper "github.com/kubeshop/testkube/pkg/mapper/testsources" ) -func (s TestkubeAPI) CreateTestSourceHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) CreateTestSourceHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to create test source" var testSource testsourcev1.TestSource @@ -42,7 +43,7 @@ func (s TestkubeAPI) CreateTestSourceHandler() fiber.Handler { } data, err := crd.GenerateYAML(crd.TemplateTestSource, []testkube.TestSourceUpsertRequest{request}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } testSource = testsourcesmapper.MapAPIToCRD(request) @@ -52,7 +53,7 @@ func (s TestkubeAPI) CreateTestSourceHandler() fiber.Handler { } } - created, err := s.TestSourcesClient.Create(&testSource, testsources.Option{Secrets: secrets}) + created, err := s.DeprecatedClients.TestSources().Create(&testSource, testsources.Option{Secrets: secrets}) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not create test source: %w", errPrefix, err)) } @@ -62,7 +63,7 @@ func (s TestkubeAPI) CreateTestSourceHandler() fiber.Handler { } } -func (s TestkubeAPI) UpdateTestSourceHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) UpdateTestSourceHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to update test source" var request testkube.TestSourceUpdateRequest @@ -88,7 +89,7 @@ func (s TestkubeAPI) UpdateTestSourceHandler() fiber.Handler { } errPrefix = errPrefix + " " + name // we need to get resource first and load its metadata.ResourceVersion - testSource, err := s.TestSourcesClient.Get(name) + testSource, err := s.DeprecatedClients.TestSources().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: test source not found: %w", errPrefix, err)) @@ -116,9 +117,9 @@ func (s TestkubeAPI) UpdateTestSourceHandler() fiber.Handler { var updatedTestSource *testsourcev1.TestSource if option != nil { - updatedTestSource, err = s.TestSourcesClient.Update(testSourceSpec, *option) + updatedTestSource, err = s.DeprecatedClients.TestSources().Update(testSourceSpec, *option) } else { - updatedTestSource, err = s.TestSourcesClient.Update(testSourceSpec) + updatedTestSource, err = s.DeprecatedClients.TestSources().Update(testSourceSpec) } if err != nil { @@ -129,11 +130,11 @@ func (s TestkubeAPI) UpdateTestSourceHandler() fiber.Handler { } } -func (s TestkubeAPI) ListTestSourcesHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ListTestSourcesHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to list test sources" - list, err := s.TestSourcesClient.List(c.Query("selector")) + list, err := s.DeprecatedClients.TestSources().List(c.Query("selector")) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not list test sources: %s", errPrefix, err)) } @@ -151,19 +152,19 @@ func (s TestkubeAPI) ListTestSourcesHandler() fiber.Handler { } data, err := crd.GenerateYAML(crd.TemplateTestSource, results) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } return c.JSON(results) } } -func (s TestkubeAPI) GetTestSourceHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) GetTestSourceHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("name") errPrefix := "failed to get test source" + name - item, err := s.TestSourcesClient.Get(name) + item, err := s.DeprecatedClients.TestSources().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: client could not find test source: %w", errPrefix, err)) @@ -178,19 +179,19 @@ func (s TestkubeAPI) GetTestSourceHandler() fiber.Handler { } data, err := crd.GenerateYAML(crd.TemplateTestSource, []testkube.TestSource{result}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } return c.JSON(result) } } -func (s TestkubeAPI) DeleteTestSourceHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) DeleteTestSourceHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("name") errPrefix := "failed to delete test source" + name - err := s.TestSourcesClient.Delete(name) + err := s.DeprecatedClients.TestSources().Delete(name) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not delete test source: %w", errPrefix, err)) } @@ -200,10 +201,10 @@ func (s TestkubeAPI) DeleteTestSourceHandler() fiber.Handler { } } -func (s TestkubeAPI) DeleteTestSourcesHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) DeleteTestSourcesHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to delete test sources" - err := s.TestSourcesClient.DeleteByLabels(c.Query("selector")) + err := s.DeprecatedClients.TestSources().DeleteByLabels(c.Query("selector")) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not delete test sources: %w", errPrefix, err)) } @@ -213,7 +214,7 @@ func (s TestkubeAPI) DeleteTestSourcesHandler() fiber.Handler { } } -func (s TestkubeAPI) ProcessTestSourceBatchHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ProcessTestSourceBatchHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to batch process test sources" @@ -232,7 +233,7 @@ func (s TestkubeAPI) ProcessTestSourceBatchHandler() fiber.Handler { testSourceBatch[item.Name] = item } - list, err := s.TestSourcesClient.List("") + list, err := s.DeprecatedClients.TestSources().List("") if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not list test sources: %w", errPrefix, err)) } @@ -254,7 +255,7 @@ func (s TestkubeAPI) ProcessTestSourceBatchHandler() fiber.Handler { if existed, ok := testSourceMap[name]; !ok { testSource.Namespace = s.Namespace - created, err := s.TestSourcesClient.Create(&testSource, testsources.Option{Secrets: getTestSourceSecretsData(username, token)}) + created, err := s.DeprecatedClients.TestSources().Create(&testSource, testsources.Option{Secrets: getTestSourceSecretsData(username, token)}) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not create test source %s: %w", errPrefix, testSource.Name, err)) } @@ -264,7 +265,7 @@ func (s TestkubeAPI) ProcessTestSourceBatchHandler() fiber.Handler { existed.Spec = testSource.Spec existed.Labels = item.Labels - updated, err := s.TestSourcesClient.Update(&existed, testsources.Option{Secrets: getTestSourceSecretsData(username, token)}) + updated, err := s.DeprecatedClients.TestSources().Update(&existed, testsources.Option{Secrets: getTestSourceSecretsData(username, token)}) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not update test source %s: %w", errPrefix, testSource.Name, err)) } @@ -275,7 +276,7 @@ func (s TestkubeAPI) ProcessTestSourceBatchHandler() fiber.Handler { for name := range testSourceMap { if _, ok := testSourceBatch[name]; !ok { - err := s.TestSourcesClient.Delete(name) + err := s.DeprecatedClients.TestSources().Delete(name) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not delete test source %s: %w", errPrefix, name, err)) } diff --git a/internal/app/api/v1/testsuites.go b/internal/app/api/deprecatedv1/testsuites.go similarity index 85% rename from internal/app/api/v1/testsuites.go rename to internal/app/api/deprecatedv1/testsuites.go index 0e7fb73f27e..a4f6a003164 100644 --- a/internal/app/api/v1/testsuites.go +++ b/internal/app/api/deprecatedv1/testsuites.go @@ -1,4 +1,4 @@ -package v1 +package deprecatedv1 import ( "bytes" @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" testsuitesv3 "github.com/kubeshop/testkube-operator/api/testsuite/v3" + "github.com/kubeshop/testkube/internal/app/api/apiutils" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/crd" "github.com/kubeshop/testkube/pkg/datefilter" @@ -33,7 +34,7 @@ import ( ) // CreateTestSuiteHandler for getting test object -func (s TestkubeAPI) CreateTestSuiteHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) CreateTestSuiteHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to create test suite" var testSuite testsuitesv3.TestSuite @@ -77,7 +78,7 @@ func (s TestkubeAPI) CreateTestSuiteHandler() fiber.Handler { if c.Accepts(mediaTypeJSON, mediaTypeYAML) == mediaTypeYAML { request.QuoteTestSuiteTextFields() data, err := crd.GenerateYAML(crd.TemplateTestSuite, []testkube.TestSuiteUpsertRequest{request}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } testSuite, err = testsuitesmapper.MapTestSuiteUpsertRequestToTestCRD(request) @@ -90,7 +91,7 @@ func (s TestkubeAPI) CreateTestSuiteHandler() fiber.Handler { s.Log.Infow("creating test suite", "testSuite", testSuite) - created, err := s.TestsSuitesClient.Create(&testSuite, !s.secretConfig.AutoCreate) + created, err := s.DeprecatedClients.TestSuites().Create(&testSuite, !s.secretConfig.AutoCreate) s.Metrics.IncCreateTestSuite(err) @@ -104,7 +105,7 @@ func (s TestkubeAPI) CreateTestSuiteHandler() fiber.Handler { } // UpdateTestSuiteHandler updates an existing TestSuite CR based on TestSuite content -func (s TestkubeAPI) UpdateTestSuiteHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) UpdateTestSuiteHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to update test suite" var request testkube.TestSuiteUpdateRequest @@ -154,7 +155,7 @@ func (s TestkubeAPI) UpdateTestSuiteHandler() fiber.Handler { errPrefix = errPrefix + " " + name // we need to get resource first and load its metadata.ResourceVersion - testSuite, err := s.TestsSuitesClient.Get(name) + testSuite, err := s.DeprecatedClients.TestSuites().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: test suite not found: %w", errPrefix, err)) @@ -169,7 +170,7 @@ func (s TestkubeAPI) UpdateTestSuiteHandler() fiber.Handler { return s.Error(c, http.StatusBadRequest, err) } - updatedTestSuite, err := s.TestsSuitesClient.Update(testSuiteSpec, !s.secretConfig.AutoCreate) + updatedTestSuite, err := s.DeprecatedClients.TestSuites().Update(testSuiteSpec, !s.secretConfig.AutoCreate) s.Metrics.IncUpdateTestSuite(err) @@ -182,12 +183,12 @@ func (s TestkubeAPI) UpdateTestSuiteHandler() fiber.Handler { } // GetTestSuiteHandler for getting TestSuite object -func (s TestkubeAPI) GetTestSuiteHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) GetTestSuiteHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("id") errPrefix := "failed to get test suite " + name - crTestSuite, err := s.TestsSuitesClient.Get(name) + crTestSuite, err := s.DeprecatedClients.TestSuites().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Warn(c, http.StatusNotFound, fmt.Errorf("%s: test suite not found: %w", errPrefix, err)) @@ -200,7 +201,7 @@ func (s TestkubeAPI) GetTestSuiteHandler() fiber.Handler { if c.Accepts(mediaTypeJSON, mediaTypeYAML) == mediaTypeYAML { testSuite.QuoteTestSuiteTextFields() data, err := crd.GenerateYAML(crd.TemplateTestSuite, []testkube.TestSuite{testSuite}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } return c.JSON(testSuite) @@ -208,11 +209,11 @@ func (s TestkubeAPI) GetTestSuiteHandler() fiber.Handler { } // GetTestSuiteWithExecutionHandler for getting TestSuite object with execution -func (s TestkubeAPI) GetTestSuiteWithExecutionHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) GetTestSuiteWithExecutionHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("id") errPrefix := fmt.Sprintf("failed to get test suite %s with execution", name) - crTestSuite, err := s.TestsSuitesClient.Get(name) + crTestSuite, err := s.DeprecatedClients.TestSuites().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Warn(c, http.StatusNotFound, fmt.Errorf("%s: test suite not found: %w", errPrefix, err)) @@ -225,11 +226,11 @@ func (s TestkubeAPI) GetTestSuiteWithExecutionHandler() fiber.Handler { if c.Accepts(mediaTypeJSON, mediaTypeYAML) == mediaTypeYAML { testSuite.QuoteTestSuiteTextFields() data, err := crd.GenerateYAML(crd.TemplateTestSuite, []testkube.TestSuite{testSuite}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } ctx := c.Context() - execution, err := s.TestExecutionResults.GetLatestByTestSuite(ctx, name) + execution, err := s.DeprecatedRepositories.TestSuiteResults().GetLatestByTestSuite(ctx, name) if err != nil && err != mongo.ErrNoDocuments { return s.Error(c, http.StatusInternalServerError, fmt.Errorf("%s: could not get execution: %w", errPrefix, err)) } @@ -242,13 +243,13 @@ func (s TestkubeAPI) GetTestSuiteWithExecutionHandler() fiber.Handler { } // DeleteTestSuiteHandler for deleting a TestSuite with id -func (s TestkubeAPI) DeleteTestSuiteHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) DeleteTestSuiteHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("id") errPrefix := fmt.Sprintf("failed to delete test suite %s", name) skipCRD := c.Query("skipDeleteCRD", "") if skipCRD != "true" { - err := s.TestsSuitesClient.Delete(name) + err := s.DeprecatedClients.TestSuites().Delete(name) if err != nil { if errors.IsNotFound(err) { return s.Warn(c, http.StatusNotFound, fmt.Errorf("%s: test suite not found: %w", errPrefix, err)) @@ -259,12 +260,12 @@ func (s TestkubeAPI) DeleteTestSuiteHandler() fiber.Handler { } // delete executions for test - if err := s.ExecutionResults.DeleteByTestSuite(c.Context(), name); err != nil { + if err := s.DeprecatedRepositories.TestResults().DeleteByTestSuite(c.Context(), name); err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not delete test suite test executions: %w", errPrefix, err)) } // delete executions for test suite - if err := s.TestExecutionResults.DeleteByTestSuite(c.Context(), name); err != nil { + if err := s.DeprecatedRepositories.TestSuiteResults().DeleteByTestSuite(c.Context(), name); err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not delete test suite executions: %w", errPrefix, err)) } @@ -273,7 +274,7 @@ func (s TestkubeAPI) DeleteTestSuiteHandler() fiber.Handler { } // DeleteTestSuitesHandler for deleting all TestSuites -func (s TestkubeAPI) DeleteTestSuitesHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) DeleteTestSuitesHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to delete test suites" @@ -281,10 +282,10 @@ func (s TestkubeAPI) DeleteTestSuitesHandler() fiber.Handler { var testSuiteNames []string selector := c.Query("selector") if selector == "" { - err = s.TestsSuitesClient.DeleteAll() + err = s.DeprecatedClients.TestSuites().DeleteAll() } else { var testSuiteList *testsuitesv3.TestSuiteList - testSuiteList, err = s.TestsSuitesClient.List(selector) + testSuiteList, err = s.DeprecatedClients.TestSuites().List(selector) if err != nil { if !errors.IsNotFound(err) { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not list test suites: %w", errPrefix, err)) @@ -295,7 +296,7 @@ func (s TestkubeAPI) DeleteTestSuitesHandler() fiber.Handler { } } - err = s.TestsSuitesClient.DeleteByLabels(selector) + err = s.DeprecatedClients.TestSuites().DeleteByLabels(selector) } if err != nil { @@ -308,9 +309,9 @@ func (s TestkubeAPI) DeleteTestSuitesHandler() fiber.Handler { // delete all executions for tests if selector == "" { - err = s.ExecutionResults.DeleteForAllTestSuites(c.Context()) + err = s.DeprecatedRepositories.TestResults().DeleteForAllTestSuites(c.Context()) } else { - err = s.ExecutionResults.DeleteByTestSuites(c.Context(), testSuiteNames) + err = s.DeprecatedRepositories.TestResults().DeleteByTestSuites(c.Context(), testSuiteNames) } if err != nil { @@ -319,9 +320,9 @@ func (s TestkubeAPI) DeleteTestSuitesHandler() fiber.Handler { // delete all executions for test suites if selector == "" { - err = s.TestExecutionResults.DeleteAll(c.Context()) + err = s.DeprecatedRepositories.TestSuiteResults().DeleteAll(c.Context()) } else { - err = s.TestExecutionResults.DeleteByTestSuites(c.Context(), testSuiteNames) + err = s.DeprecatedRepositories.TestSuiteResults().DeleteByTestSuites(c.Context(), testSuiteNames) } if err != nil { @@ -332,8 +333,8 @@ func (s TestkubeAPI) DeleteTestSuitesHandler() fiber.Handler { } } -func (s TestkubeAPI) getFilteredTestSuitesList(c *fiber.Ctx) (*testsuitesv3.TestSuiteList, error) { - crTestSuites, err := s.TestsSuitesClient.List(c.Query("selector")) +func (s *DeprecatedTestkubeAPI) getFilteredTestSuitesList(c *fiber.Ctx) (*testsuitesv3.TestSuiteList, error) { + crTestSuites, err := s.DeprecatedClients.TestSuites().List(c.Query("selector")) if err != nil { return nil, err } @@ -352,7 +353,7 @@ func (s TestkubeAPI) getFilteredTestSuitesList(c *fiber.Ctx) (*testsuitesv3.Test } // ListTestSuitesHandler for getting list of all available TestSuites -func (s TestkubeAPI) ListTestSuitesHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ListTestSuitesHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to list test suites" @@ -368,7 +369,7 @@ func (s TestkubeAPI) ListTestSuitesHandler() fiber.Handler { } data, err := crd.GenerateYAML(crd.TemplateTestSuite, testSuites) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } return c.JSON(testSuites) @@ -376,7 +377,7 @@ func (s TestkubeAPI) ListTestSuitesHandler() fiber.Handler { } // TestSuiteMetricsHandler returns basic metrics for given testsuite -func (s TestkubeAPI) TestSuiteMetricsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) TestSuiteMetricsHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to get test suite metrics" const ( @@ -396,7 +397,7 @@ func (s TestkubeAPI) TestSuiteMetricsHandler() fiber.Handler { last = DefaultLastDays } - metrics, err := s.TestExecutionResults.GetTestSuiteMetrics(context.Background(), testSuiteName, limit, last) + metrics, err := s.DeprecatedRepositories.TestSuiteResults().GetTestSuiteMetrics(context.Background(), testSuiteName, limit, last) if err != nil { return s.Error(c, http.StatusInternalServerError, fmt.Errorf("%s: failed to get metrics from client: %w", errPrefix, err)) } @@ -406,8 +407,8 @@ func (s TestkubeAPI) TestSuiteMetricsHandler() fiber.Handler { } // getLatestTestSuiteExecutions return latest test suite executions either by starttime or endtine for tests -func (s TestkubeAPI) getLatestTestSuiteExecutions(ctx context.Context, testSuiteNames []string) (map[string]testkube.TestSuiteExecution, error) { - executions, err := s.TestExecutionResults.GetLatestByTestSuites(ctx, testSuiteNames) +func (s *DeprecatedTestkubeAPI) getLatestTestSuiteExecutions(ctx context.Context, testSuiteNames []string) (map[string]testkube.TestSuiteExecution, error) { + executions, err := s.DeprecatedRepositories.TestSuiteResults().GetLatestByTestSuites(ctx, testSuiteNames) if err != nil && err != mongo.ErrNoDocuments { return nil, err } @@ -423,7 +424,7 @@ func (s TestkubeAPI) getLatestTestSuiteExecutions(ctx context.Context, testSuite } // ListTestSuiteWithExecutionsHandler for getting list of all available TestSuite with latest executions -func (s TestkubeAPI) ListTestSuiteWithExecutionsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ListTestSuiteWithExecutionsHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to list test suites with executions" @@ -439,7 +440,7 @@ func (s TestkubeAPI) ListTestSuiteWithExecutionsHandler() fiber.Handler { } data, err := crd.GenerateYAML(crd.TemplateTestSuite, testSuites) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } results := make([]testkube.TestSuiteWithExecutionSummary, 0, len(testSuites)) @@ -540,7 +541,7 @@ func (s TestkubeAPI) ListTestSuiteWithExecutionsHandler() fiber.Handler { } } -func (s TestkubeAPI) ExecuteTestSuitesHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ExecuteTestSuitesHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to execute test suite" var request testkube.TestSuiteExecutionRequest @@ -556,7 +557,7 @@ func (s TestkubeAPI) ExecuteTestSuitesHandler() fiber.Handler { var testSuites []testsuitesv3.TestSuite if name != "" { errPrefix = errPrefix + " " + name - testSuite, err := s.TestsSuitesClient.Get(name) + testSuite, err := s.DeprecatedClients.TestSuites().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Warn(c, http.StatusNotFound, fmt.Errorf("%s: test suite not found: %w", errPrefix, err)) @@ -566,7 +567,7 @@ func (s TestkubeAPI) ExecuteTestSuitesHandler() fiber.Handler { } testSuites = append(testSuites, *testSuite) } else { - testSuiteList, err := s.TestsSuitesClient.List(selector) + testSuiteList, err := s.DeprecatedClients.TestSuites().List(selector) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: can't list test suites: %w", errPrefix, err)) } @@ -607,7 +608,7 @@ func (s TestkubeAPI) ExecuteTestSuitesHandler() fiber.Handler { } } -func (s TestkubeAPI) ListTestSuiteExecutionsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ListTestSuiteExecutionsHandler() fiber.Handler { return func(c *fiber.Ctx) error { now := time.Now() @@ -617,20 +618,20 @@ func (s TestkubeAPI) ListTestSuiteExecutionsHandler() fiber.Handler { filter := getExecutionsFilterFromRequest(c) ctx := c.Context() - executionsTotals, err := s.TestExecutionResults.GetExecutionsTotals(ctx, filter) + executionsTotals, err := s.DeprecatedRepositories.TestSuiteResults().GetExecutionsTotals(ctx, filter) if err != nil { return s.Error(c, http.StatusInternalServerError, fmt.Errorf("%s: client could not get executions totals: %w", errPrefix, err)) } l.Debugw("got executions totals", "totals", executionsTotals, "time", time.Since(now)) filterAllTotals := *filter.(*testresult.FilterImpl) filterAllTotals.WithPage(0).WithPageSize(math.MaxInt64) - allExecutionsTotals, err := s.TestExecutionResults.GetExecutionsTotals(ctx, filterAllTotals) + allExecutionsTotals, err := s.DeprecatedRepositories.TestSuiteResults().GetExecutionsTotals(ctx, filterAllTotals) if err != nil { return s.Error(c, http.StatusInternalServerError, fmt.Errorf("%s: client could not get all executions totals: %w", errPrefix, err)) } l.Debugw("got all executions totals", "totals", executionsTotals, "time", time.Since(now)) - executions, err := s.TestExecutionResults.GetExecutions(ctx, filter) + executions, err := s.DeprecatedRepositories.TestSuiteResults().GetExecutions(ctx, filter) if err != nil { return s.Error(c, http.StatusInternalServerError, fmt.Errorf("%s: client could not get executions: %w", errPrefix, err)) } @@ -644,12 +645,12 @@ func (s TestkubeAPI) ListTestSuiteExecutionsHandler() fiber.Handler { } } -func (s TestkubeAPI) GetTestSuiteExecutionHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) GetTestSuiteExecutionHandler() fiber.Handler { return func(c *fiber.Ctx) error { id := c.Params("executionID") errPrefix := fmt.Sprintf("failed to get test suite execution %s", id) - execution, err := s.TestExecutionResults.Get(c.Context(), id) + execution, err := s.DeprecatedRepositories.TestSuiteResults().Get(c.Context(), id) if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: test suite with execution id/name %s not found", errPrefix, id)) } @@ -661,7 +662,7 @@ func (s TestkubeAPI) GetTestSuiteExecutionHandler() fiber.Handler { secretMap := make(map[string]string) if execution.SecretUUID != "" && execution.TestSuite != nil { - secretMap, err = s.TestsSuitesClient.GetSecretTestSuiteVars(execution.TestSuite.Name, execution.SecretUUID) + secretMap, err = s.DeprecatedClients.TestSuites().GetSecretTestSuiteVars(execution.TestSuite.Name, execution.SecretUUID) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not get test suite secrets: %w", errPrefix, err)) } @@ -679,12 +680,12 @@ func (s TestkubeAPI) GetTestSuiteExecutionHandler() fiber.Handler { } } -func (s TestkubeAPI) ListTestSuiteArtifactsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ListTestSuiteArtifactsHandler() fiber.Handler { return func(c *fiber.Ctx) error { s.Log.Infow("listing testsuite artifacts", "executionID", c.Params("executionID")) id := c.Params("executionID") errPrefix := fmt.Sprintf("failed to list test suite artifacts %s", id) - execution, err := s.TestExecutionResults.Get(c.Context(), id) + execution, err := s.DeprecatedRepositories.TestSuiteResults().Get(c.Context(), id) if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: test suite with execution id/name %s not found", errPrefix, id)) } @@ -725,7 +726,7 @@ func (s TestkubeAPI) ListTestSuiteArtifactsHandler() fiber.Handler { } } -func (s TestkubeAPI) getExecutionArtfacts(ctx context.Context, execution *testkube.Execution, +func (s *DeprecatedTestkubeAPI) getExecutionArtfacts(ctx context.Context, execution *testkube.Execution, artifacts []testkube.Artifact) ([]testkube.Artifact, error) { var stepArtifacts []testkube.Artifact var bucket string @@ -764,7 +765,7 @@ func (s TestkubeAPI) getExecutionArtfacts(ctx context.Context, execution *testku } // AbortTestSuiteHandler for aborting a TestSuite with id -func (s TestkubeAPI) AbortTestSuiteHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) AbortTestSuiteHandler() fiber.Handler { return func(c *fiber.Ctx) error { ctx := c.Context() name := c.Params("id") @@ -773,7 +774,7 @@ func (s TestkubeAPI) AbortTestSuiteHandler() fiber.Handler { } errPrefix := fmt.Sprintf("failed to abort test suite %s", name) filter := testresult.NewExecutionsFilter().WithName(name).WithStatus(string(testkube.RUNNING_ExecutionStatus)) - executions, err := s.TestExecutionResults.GetExecutions(ctx, filter) + executions, err := s.DeprecatedRepositories.TestSuiteResults().GetExecutions(ctx, filter) if err != nil { if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: executions with test syute name %s not found", errPrefix, name)) @@ -797,12 +798,12 @@ func (s TestkubeAPI) AbortTestSuiteHandler() fiber.Handler { } } -func (s TestkubeAPI) AbortTestSuiteExecutionHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) AbortTestSuiteExecutionHandler() fiber.Handler { return func(c *fiber.Ctx) error { s.Log.Infow("aborting test suite execution", "executionID", c.Params("executionID")) id := c.Params("executionID") errPrefix := fmt.Sprintf("failed to abort test suite execution %s", id) - execution, err := s.TestExecutionResults.Get(c.Context(), id) + execution, err := s.DeprecatedRepositories.TestSuiteResults().Get(c.Context(), id) if err == mongo.ErrNoDocuments { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: test suite with execution id/name %s not found", errPrefix, id)) } @@ -825,11 +826,11 @@ func (s TestkubeAPI) AbortTestSuiteExecutionHandler() fiber.Handler { } // ListTestSuiteTestsHandler for getting list of all available Tests for TestSuites -func (s TestkubeAPI) ListTestSuiteTestsHandler() fiber.Handler { +func (s *DeprecatedTestkubeAPI) ListTestSuiteTestsHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("id") errPrefix := fmt.Sprintf("failed to list tests for test suite %s", name) - crTestSuite, err := s.TestsSuitesClient.Get(name) + crTestSuite, err := s.DeprecatedClients.TestSuites().Get(name) if err != nil { if errors.IsNotFound(err) { return s.Warn(c, http.StatusNotFound, fmt.Errorf("%s: test suite with id/name %s not found", errPrefix, name)) @@ -839,7 +840,7 @@ func (s TestkubeAPI) ListTestSuiteTestsHandler() fiber.Handler { } testSuite := testsuitesmapper.MapCRToAPI(*crTestSuite) - crTests, err := s.TestsClient.ListByNames(testSuite.GetTestNames()) + crTests, err := s.DeprecatedClients.Tests().ListByNames(testSuite.GetTestNames()) if err != nil { if errors.IsNotFound(err) { return s.Warn(c, http.StatusNotFound, fmt.Errorf("%s: test suite tests with id/name %s not found", errPrefix, testSuite.GetTestNames())) diff --git a/internal/app/api/v1/uploads.go b/internal/app/api/deprecatedv1/uploads.go similarity index 94% rename from internal/app/api/v1/uploads.go rename to internal/app/api/deprecatedv1/uploads.go index 6508fbca088..793f1f74c2d 100644 --- a/internal/app/api/v1/uploads.go +++ b/internal/app/api/deprecatedv1/uploads.go @@ -1,4 +1,4 @@ -package v1 +package deprecatedv1 import ( "fmt" @@ -7,7 +7,7 @@ import ( ) // UploadFiles uploads files into the object store and uses them during execution -func (s TestkubeAPI) UploadFiles() fiber.Handler { +func (s *DeprecatedTestkubeAPI) UploadFiles() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to upload file" diff --git a/internal/app/api/v1/uploads_test.go b/internal/app/api/deprecatedv1/uploads_test.go similarity index 94% rename from internal/app/api/v1/uploads_test.go rename to internal/app/api/deprecatedv1/uploads_test.go index 6fad514363a..afd0d976e65 100644 --- a/internal/app/api/v1/uploads_test.go +++ b/internal/app/api/deprecatedv1/uploads_test.go @@ -1,4 +1,4 @@ -package v1 +package deprecatedv1 import ( "bytes" @@ -17,10 +17,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/log" - "github.com/kubeshop/testkube/pkg/server" ) -func TestTestkubeAPI_UploadCopyFiles(t *testing.T) { +func TestDeprecatedTestkubeAPI_UploadCopyFiles(t *testing.T) { t.Parallel() mockCtrl := gomock.NewController(t) @@ -29,11 +28,8 @@ func TestTestkubeAPI_UploadCopyFiles(t *testing.T) { mockArtifactsStorage := storage.NewMockArtifactsStorage(mockCtrl) app := fiber.New() - s := &TestkubeAPI{ - HTTPServer: server.HTTPServer{ - Mux: app, - Log: log.DefaultLogger, - }, + s := &DeprecatedTestkubeAPI{ + Log: log.DefaultLogger, ArtifactsStorage: mockArtifactsStorage, } route := "/uploads" diff --git a/internal/app/api/oauth/oauth.go b/internal/app/api/oauth/oauth.go new file mode 100644 index 00000000000..d70c2103438 --- /dev/null +++ b/internal/app/api/oauth/oauth.go @@ -0,0 +1,45 @@ +package oauth + +import ( + "net/http" + "strings" + + "github.com/gofiber/fiber/v2" + + "github.com/kubeshop/testkube/internal/app/api/apiutils" + "github.com/kubeshop/testkube/pkg/log" + "github.com/kubeshop/testkube/pkg/oauth" +) + +const ( + // cliIngressHeader is cli ingress header + cliIngressHeader = "X-CLI-Ingress" +) + +type OauthParams struct { + ClientID string + ClientSecret string + Provider oauth.ProviderType + Scopes string +} + +// CreateOAuthHandler is auth middleware +func CreateOAuthHandler(oauthParams OauthParams) fiber.Handler { + return func(c *fiber.Ctx) error { + if c.Get(cliIngressHeader, "") != "" { + token := strings.TrimSpace(strings.TrimPrefix(c.Get("Authorization", ""), oauth.AuthorizationPrefix)) + var scopes []string + if oauthParams.Scopes != "" { + scopes = strings.Split(oauthParams.Scopes, ",") + } + + provider := oauth.NewProvider(oauthParams.ClientID, oauthParams.ClientSecret, scopes) + if err := provider.ValidateAccessToken(oauthParams.Provider, token); err != nil { + log.DefaultLogger.Errorw("error validating token", "error", err) + return apiutils.SendError(c, http.StatusUnauthorized, err) + } + } + + return c.Next() + } +} diff --git a/internal/app/api/v1/.keep b/internal/app/api/v1/.keep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/internal/app/api/v1/configs.go b/internal/app/api/v1/configs.go index 9e6e141fdce..06fa97351c5 100644 --- a/internal/app/api/v1/configs.go +++ b/internal/app/api/v1/configs.go @@ -11,7 +11,7 @@ import ( ) // GetConfigsHandler returns configuration -func (s TestkubeAPI) GetConfigsHandler() fiber.Handler { +func (s *TestkubeAPI) GetConfigsHandler() fiber.Handler { return func(c *fiber.Ctx) error { ctx := c.Context() config, err := s.ConfigMap.Get(ctx) @@ -26,7 +26,7 @@ func (s TestkubeAPI) GetConfigsHandler() fiber.Handler { } // UpdateConfigsHandler update configuration handler -func (s TestkubeAPI) UpdateConfigsHandler() fiber.Handler { +func (s *TestkubeAPI) UpdateConfigsHandler() fiber.Handler { return func(c *fiber.Ctx) error { ctx := c.Context() errPrefix := "failed to update config" diff --git a/internal/app/api/v1/crd.go b/internal/app/api/v1/crd.go deleted file mode 100644 index c4d73d6ce75..00000000000 --- a/internal/app/api/v1/crd.go +++ /dev/null @@ -1,17 +0,0 @@ -package v1 - -import ( - "fmt" - "net/http" - - "github.com/gofiber/fiber/v2" -) - -func (s TestkubeAPI) getCRDs(c *fiber.Ctx, data string, err error) error { - if err != nil { - return s.Error(c, http.StatusBadRequest, fmt.Errorf("could not build CRD: %w", err)) - } - - c.Context().SetContentType(mediaTypeYAML) - return c.SendString(data) -} diff --git a/internal/app/api/v1/debug.go b/internal/app/api/v1/debug.go index f09286b7fa2..e81eed48d30 100644 --- a/internal/app/api/v1/debug.go +++ b/internal/app/api/v1/debug.go @@ -5,7 +5,7 @@ import ( ) // GetDebugListenersHandler returns event logs -func (s TestkubeAPI) GetDebugListenersHandler() fiber.Handler { +func (s *TestkubeAPI) GetDebugListenersHandler() fiber.Handler { return func(c *fiber.Ctx) error { return c.JSON(s.Events.Listeners.Log()) } diff --git a/internal/app/api/v1/events.go b/internal/app/api/v1/events.go index 91ce2fde15e..0646d8f0b94 100644 --- a/internal/app/api/v1/events.go +++ b/internal/app/api/v1/events.go @@ -1,7 +1,6 @@ package v1 import ( - "context" "encoding/json" "net/http" @@ -11,16 +10,7 @@ import ( events "github.com/fluxcd/pkg/apis/event/v1beta1" ) -// InitEvents is a handler to emit logs -func (s TestkubeAPI) InitEvents() { - // run reconciller loop - go s.Events.Reconcile(context.Background()) - - // run workers - s.Events.Listen(context.Background()) -} - -func (s TestkubeAPI) EventsStreamHandler() fiber.Handler { +func (s *TestkubeAPI) EventsStreamHandler() fiber.Handler { return websocket.New(func(c *websocket.Conn) { s.Log.Debugw("handling websocket connection", "id", c.Params("id"), "remoteAddr", c.RemoteAddr(), "localAddr", c.LocalAddr()) @@ -33,7 +23,7 @@ func (s TestkubeAPI) EventsStreamHandler() fiber.Handler { } // GetTestHandler is method for getting an existing test -func (s TestkubeAPI) FluxEventHandler() fiber.Handler { +func (s *TestkubeAPI) FluxEventHandler() fiber.Handler { return func(c *fiber.Ctx) error { body := c.Body() diff --git a/internal/app/api/v1/events_test.go b/internal/app/api/v1/events_test.go index 87705c57dc6..2ac79922793 100644 --- a/internal/app/api/v1/events_test.go +++ b/internal/app/api/v1/events_test.go @@ -8,20 +8,16 @@ import ( "github.com/gofiber/fiber/v2" - "github.com/kubeshop/testkube/pkg/log" - "github.com/kubeshop/testkube/pkg/server" - "github.com/stretchr/testify/assert" + + "github.com/kubeshop/testkube/pkg/log" ) func TestTestkubeAPI_FluxEventHandler(t *testing.T) { // bootstrap api server fiber app app := fiber.New() s := &TestkubeAPI{ - HTTPServer: server.HTTPServer{ - Mux: app, - Log: log.DefaultLogger, - }, + Log: log.DefaultLogger, } app.Post("/events/flux", s.FluxEventHandler()) diff --git a/internal/app/api/v1/handlers.go b/internal/app/api/v1/handlers.go index cdcf2974d44..f6e46002ed4 100644 --- a/internal/app/api/v1/handlers.go +++ b/internal/app/api/v1/handlers.go @@ -3,20 +3,17 @@ package v1 import ( "fmt" "net/http" - "strings" "github.com/gofiber/fiber/v2" "github.com/pkg/errors" + "github.com/kubeshop/testkube/internal/app/api/apiutils" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/k8sclient" - "github.com/kubeshop/testkube/pkg/oauth" "github.com/kubeshop/testkube/pkg/version" ) const ( - // cliIngressHeader is cli ingress header - cliIngressHeader = "X-CLI-Ingress" // mediaTypeJSON is json media type mediaTypeJSON = "application/json" // mediaTypeYAML is yaml media type @@ -31,27 +28,6 @@ const ( contextOSS = "oss" ) -// AuthHandler is auth middleware -func (s *TestkubeAPI) AuthHandler() fiber.Handler { - return func(c *fiber.Ctx) error { - if c.Get(cliIngressHeader, "") != "" { - token := strings.TrimSpace(strings.TrimPrefix(c.Get("Authorization", ""), oauth.AuthorizationPrefix)) - var scopes []string - if s.oauthParams.Scopes != "" { - scopes = strings.Split(s.oauthParams.Scopes, ",") - } - - provider := oauth.NewProvider(s.oauthParams.ClientID, s.oauthParams.ClientSecret, scopes) - if err := provider.ValidateAccessToken(s.oauthParams.Provider, token); err != nil { - s.Log.Errorw("error validating token", "error", err) - return s.Error(c, http.StatusUnauthorized, err) - } - } - - return c.Next() - } -} - // InfoHandler is a handler to get info func (s *TestkubeAPI) InfoHandler() fiber.Handler { apiContext := contextOSS @@ -79,7 +55,7 @@ func (s *TestkubeAPI) InfoHandler() fiber.Handler { Version: version.Version, Namespace: s.Namespace, Context: apiContext, - ClusterId: s.Config.ClusterID, + ClusterId: s.ClusterID, EnvId: envID, OrgId: orgID, HelmchartVersion: s.helmchartVersion, @@ -96,23 +72,6 @@ func (s *TestkubeAPI) InfoHandler() fiber.Handler { } } -// RoutesHandler is a handler to get existing routes -func (s *TestkubeAPI) RoutesHandler() fiber.Handler { - return func(c *fiber.Ctx) error { - var routes []fiber.Route - - stack := s.Mux.Stack() - for _, e := range stack { - for _, s := range e { - route := *s - routes = append(routes, route) - } - } - - return c.JSON(routes) - } -} - // DebugHandler is a handler to get debug information func (s *TestkubeAPI) DebugHandler() fiber.Handler { return func(c *fiber.Ctx) error { @@ -137,20 +96,24 @@ func (s *TestkubeAPI) DebugHandler() fiber.Handler { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: could not get operator logs: %w", errPrefix, err)) } - executionLogs, err := s.GetLatestExecutionLogs(c.UserContext()) - if err != nil { - return s.Error(c, http.StatusInternalServerError, fmt.Errorf("%s: could not get execution logs: %w", errPrefix, err)) - } - return c.JSON(testkube.DebugInfo{ ClusterVersion: clusterVersion, ApiLogs: apiLogs, OperatorLogs: operatorLogs, - ExecutionLogs: executionLogs, }) } } +// Warn writes RFC-7807 json problem to response +func (s *TestkubeAPI) Warn(c *fiber.Ctx, status int, err error, context ...interface{}) error { + return apiutils.SendWarn(c, status, err, context...) +} + +// Error writes RFC-7807 json problem to response +func (s *TestkubeAPI) Error(c *fiber.Ctx, status int, err error, context ...interface{}) error { + return apiutils.SendError(c, status, err, context...) +} + func (s *TestkubeAPI) NotImplemented(c *fiber.Ctx) error { return s.Error(c, http.StatusNotImplemented, errors.New("not implemented yet")) } @@ -172,7 +135,7 @@ func (s *TestkubeAPI) NotFound(c *fiber.Ctx, prefix, description string, err err } func (s *TestkubeAPI) ClientError(c *fiber.Ctx, prefix string, err error) error { - if IsNotFound(err) { + if apiutils.IsNotFound(err) { return s.NotFound(c, prefix, "client not found", err) } return s.BadGateway(c, prefix, "client problem", err) diff --git a/internal/app/api/v1/labels_tags.go b/internal/app/api/v1/labels_tags.go index 2e3271df8c3..3007bc2d4e8 100644 --- a/internal/app/api/v1/labels_tags.go +++ b/internal/app/api/v1/labels_tags.go @@ -7,10 +7,17 @@ import ( "github.com/gofiber/fiber/v2" ) -func (s TestkubeAPI) ListLabelsHandler() fiber.Handler { +type LabelSource interface { + ListLabels() (map[string][]string, error) +} + +func (s *TestkubeAPI) ListLabelsHandler() fiber.Handler { return func(c *fiber.Ctx) error { labels := make(map[string][]string) - sources := append(*s.LabelSources, s.TestsClient, s.TestsSuitesClient) + sources := []LabelSource{s.TestWorkflowsClient, s.TestWorkflowTemplatesClient} + if s.DeprecatedClients != nil { + sources = append(sources, s.DeprecatedClients.Tests(), s.DeprecatedClients.TestSuites()) + } for _, source := range sources { nextLabels, err := source.ListLabels() diff --git a/internal/app/api/v1/repository.go b/internal/app/api/v1/repository.go index e4663396478..8de13a49a22 100644 --- a/internal/app/api/v1/repository.go +++ b/internal/app/api/v1/repository.go @@ -31,7 +31,7 @@ var ( errPathNotFound = errors.New("path: The specified path could not be found") ) -func (s TestkubeAPI) ValidateRepositoryHandler() fiber.Handler { +func (s *TestkubeAPI) ValidateRepositoryHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to validate repository" var request testkube.Repository diff --git a/internal/app/api/v1/secret.go b/internal/app/api/v1/secret.go index 4d63fe596ae..37ccf905db0 100644 --- a/internal/app/api/v1/secret.go +++ b/internal/app/api/v1/secret.go @@ -10,6 +10,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" + "github.com/kubeshop/testkube/internal/app/api/apiutils" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/secretmanager" ) @@ -101,7 +102,7 @@ func (s *TestkubeAPI) DeleteSecretHandler() fiber.Handler { err := s.SecretManager.Delete(c.Context(), namespace, name) if errors.Is(err, secretmanager.ErrDeleteDisabled) { return s.Error(c, http.StatusForbidden, errors.Wrap(err, errPrefix)) - } else if IsNotFound(err) { + } else if apiutils.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: secret not found", errPrefix)) } else if err != nil { return s.Error(c, http.StatusBadGateway, errors.Wrap(err, errPrefix)) @@ -158,7 +159,7 @@ func (s *TestkubeAPI) UpdateSecretHandler() fiber.Handler { }) if errors.Is(err, secretmanager.ErrModifyDisabled) { return s.Error(c, http.StatusForbidden, errors.Wrap(err, errPrefix)) - } else if IsNotFound(err) { + } else if apiutils.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: secret not found", errPrefix)) } else if err != nil { return s.Error(c, http.StatusBadGateway, errors.Wrap(err, errPrefix)) @@ -180,7 +181,7 @@ func (s *TestkubeAPI) GetSecretHandler() fiber.Handler { // Get the secret details secret, err := s.SecretManager.Get(c.Context(), namespace, name) - if IsNotFound(err) { + if apiutils.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: secret not found", errPrefix)) } else if errors.Is(err, secretmanager.ErrManagementDisabled) { return s.Error(c, http.StatusForbidden, errors.Wrap(err, errPrefix)) diff --git a/internal/app/api/v1/server.go b/internal/app/api/v1/server.go index bd81cdbae50..e555f095ecd 100644 --- a/internal/app/api/v1/server.go +++ b/internal/app/api/v1/server.go @@ -1,334 +1,122 @@ package v1 import ( - "context" - "io" - "net" - "os" - "reflect" - "strconv" - "sync" - "syscall" - "time" - - "github.com/pkg/errors" + "go.uber.org/zap" + testtriggersclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testtriggers/v1" testworkflowsv1 "github.com/kubeshop/testkube-operator/pkg/client/testworkflows/v1" - "github.com/kubeshop/testkube/internal/common" + "github.com/kubeshop/testkube/cmd/api-server/commons" "github.com/kubeshop/testkube/internal/config" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/log" repoConfig "github.com/kubeshop/testkube/pkg/repository/config" "github.com/kubeshop/testkube/pkg/repository/testworkflow" "github.com/kubeshop/testkube/pkg/secretmanager" "github.com/kubeshop/testkube/pkg/testworkflows/executionworker/executionworkertypes" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowexecutor" - "github.com/kubeshop/testkube/pkg/version" - - "github.com/kubeshop/testkube/pkg/datefilter" - "github.com/kubeshop/testkube/pkg/repository/result" - "github.com/kubeshop/testkube/pkg/repository/testresult" - - "k8s.io/client-go/kubernetes" - - "github.com/gofiber/fiber/v2" - "github.com/gofiber/fiber/v2/middleware/cors" - "github.com/gofiber/fiber/v2/middleware/proxy" - "github.com/kelseyhightower/envconfig" - executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" - templatesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1" - testsclientv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" - testsourcesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testsources/v1" - testsuitesclientv3 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v3" - testkubeclientset "github.com/kubeshop/testkube-operator/pkg/clientset/versioned" "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/pkg/event" - "github.com/kubeshop/testkube/pkg/event/bus" - "github.com/kubeshop/testkube/pkg/event/kind/cdevent" - "github.com/kubeshop/testkube/pkg/event/kind/k8sevent" - "github.com/kubeshop/testkube/pkg/event/kind/slack" - "github.com/kubeshop/testkube/pkg/event/kind/webhook" ws "github.com/kubeshop/testkube/pkg/event/kind/websocket" "github.com/kubeshop/testkube/pkg/executor/client" "github.com/kubeshop/testkube/pkg/featureflags" - logsclient "github.com/kubeshop/testkube/pkg/logs/client" - "github.com/kubeshop/testkube/pkg/oauth" - "github.com/kubeshop/testkube/pkg/scheduler" - "github.com/kubeshop/testkube/pkg/secret" "github.com/kubeshop/testkube/pkg/server" "github.com/kubeshop/testkube/pkg/storage" - "github.com/kubeshop/testkube/pkg/telemetry" - "github.com/kubeshop/testkube/pkg/utils/text" -) - -const ( - HeartbeatInterval = time.Hour - DefaultHttpBodyLimit = 1 * 1024 * 1024 * 1024 // 1GB - needed for file uploads ) func NewTestkubeAPI( + deprecatedClients commons.DeprecatedClients, + clusterId string, namespace string, - testExecutionResults result.Repository, - testSuiteExecutionsResults testresult.Repository, testWorkflowResults testworkflow.Repository, testWorkflowOutput testworkflow.OutputRepository, - testsClient *testsclientv3.TestsClient, - executorsClient *executorsclientv1.ExecutorsClient, - testsuitesClient *testsuitesclientv3.TestSuitesClient, - secretClient *secret.Client, - secretManager secretmanager.SecretManager, - webhookClient *executorsclientv1.WebhooksClient, - clientset kubernetes.Interface, - testkubeClientset testkubeclientset.Interface, - testsourcesClient *testsourcesclientv1.TestSourcesClient, + artifactsStorage storage.ArtifactsStorage, + webhookClient executorsclientv1.WebhooksInterface, + testTriggersClient testtriggersclientv1.Interface, testWorkflowsClient testworkflowsv1.Interface, testWorkflowTemplatesClient testworkflowsv1.TestWorkflowTemplatesInterface, configMap repoConfig.Repository, - clusterId string, - eventsEmitter *event.Emitter, - executor client.Executor, - containerExecutor client.Executor, + secretManager secretmanager.SecretManager, + secretConfig testkube.SecretConfig, testWorkflowExecutor testworkflowexecutor.TestWorkflowExecutor, executionWorkerClient executionworkertypes.Worker, + eventsEmitter *event.Emitter, + websocketLoader *ws.WebsocketLoader, metrics metrics.Metrics, - scheduler *scheduler.Scheduler, - slackLoader *slack.SlackLoader, - graphqlPort string, - artifactsStorage storage.ArtifactsStorage, - templatesClient *templatesclientv1.TemplatesClient, + proContext *config.ProContext, + ff featureflags.FeatureFlags, dashboardURI string, helmchartVersion string, - mode string, - eventsBus bus.Bus, - secretConfig testkube.SecretConfig, - ff featureflags.FeatureFlags, - logsStream logsclient.Stream, - logGrpcClient logsclient.StreamGetter, serviceAccountNames map[string]string, - envs map[string]string, dockerImageVersion string, ) TestkubeAPI { - var httpConfig server.Config - err := envconfig.Process("APISERVER", &httpConfig) - // Do we want to panic here or just ignore the error - if err != nil { - panic(err) - } - - httpConfig.ClusterID = clusterId - httpConfig.Http.BodyLimit = httpConfig.HttpBodyLimit - if httpConfig.HttpBodyLimit == 0 { - httpConfig.Http.BodyLimit = DefaultHttpBodyLimit - } - return TestkubeAPI{ - HTTPServer: server.NewServer(httpConfig), - TestExecutionResults: testSuiteExecutionsResults, - ExecutionResults: testExecutionResults, + ClusterID: clusterId, + Log: log.DefaultLogger, + DeprecatedClients: deprecatedClients, TestWorkflowResults: testWorkflowResults, TestWorkflowOutput: testWorkflowOutput, - TestsClient: testsClient, - ExecutorsClient: executorsClient, - SecretClient: secretClient, SecretManager: secretManager, - Clientset: clientset, - TestsSuitesClient: testsuitesClient, - TestKubeClientset: testkubeClientset, + TestTriggersClient: testTriggersClient, TestWorkflowsClient: testWorkflowsClient, TestWorkflowTemplatesClient: testWorkflowTemplatesClient, Metrics: metrics, + WebsocketLoader: websocketLoader, Events: eventsEmitter, WebhooksClient: webhookClient, - TestSourcesClient: testsourcesClient, Namespace: namespace, ConfigMap: configMap, - Executor: executor, - ContainerExecutor: containerExecutor, TestWorkflowExecutor: testWorkflowExecutor, ExecutionWorkerClient: executionWorkerClient, - scheduler: scheduler, - slackLoader: slackLoader, - graphqlPort: graphqlPort, ArtifactsStorage: artifactsStorage, - TemplatesClient: templatesClient, dashboardURI: dashboardURI, helmchartVersion: helmchartVersion, - mode: mode, - eventsBus: eventsBus, secretConfig: secretConfig, featureFlags: ff, - logsStream: logsStream, - logGrpcClient: logGrpcClient, - LabelSources: common.Ptr(make([]LabelSource, 0)), ServiceAccountNames: serviceAccountNames, - Envs: envs, dockerImageVersion: dockerImageVersion, + proContext: proContext, } } type TestkubeAPI struct { - server.HTTPServer - ExecutionResults result.Repository - TestExecutionResults testresult.Repository + ClusterID string + Log *zap.SugaredLogger TestWorkflowResults testworkflow.Repository TestWorkflowOutput testworkflow.OutputRepository Executor client.Executor ContainerExecutor client.Executor TestWorkflowExecutor testworkflowexecutor.TestWorkflowExecutor ExecutionWorkerClient executionworkertypes.Worker - TestsSuitesClient *testsuitesclientv3.TestSuitesClient - TestsClient *testsclientv3.TestsClient - ExecutorsClient *executorsclientv1.ExecutorsClient - SecretClient *secret.Client + DeprecatedClients commons.DeprecatedClients SecretManager secretmanager.SecretManager - WebhooksClient *executorsclientv1.WebhooksClient - TestKubeClientset testkubeclientset.Interface - TestSourcesClient *testsourcesclientv1.TestSourcesClient + WebhooksClient executorsclientv1.WebhooksInterface + TestTriggersClient testtriggersclientv1.Interface TestWorkflowsClient testworkflowsv1.Interface TestWorkflowTemplatesClient testworkflowsv1.TestWorkflowTemplatesInterface Metrics metrics.Metrics - storageParams storageParams Namespace string - oauthParams oauthParams WebsocketLoader *ws.WebsocketLoader Events *event.Emitter ConfigMap repoConfig.Repository - scheduler *scheduler.Scheduler - Clientset kubernetes.Interface - slackLoader *slack.SlackLoader - graphqlPort string ArtifactsStorage storage.ArtifactsStorage - TemplatesClient *templatesclientv1.TemplatesClient dashboardURI string helmchartVersion string - mode string - eventsBus bus.Bus secretConfig testkube.SecretConfig featureFlags featureflags.FeatureFlags - logsStream logsclient.Stream - logGrpcClient logsclient.StreamGetter proContext *config.ProContext - LabelSources *[]LabelSource ServiceAccountNames map[string]string - Envs map[string]string dockerImageVersion string } -type storageParams struct { - SSL bool `envconfig:"STORAGE_SSL" default:"false"` - SkipVerify bool `envconfig:"STORAGE_SKIP_VERIFY" default:"false"` - CertFile string `envconfig:"STORAGE_CERT_FILE"` - KeyFile string `envconfig:"STORAGE_KEY_FILE"` - CAFile string `envconfig:"STORAGE_CA_FILE"` - Endpoint string - AccessKeyId string - SecretAccessKey string - Region string - Token string - Bucket string -} - -type oauthParams struct { - ClientID string - ClientSecret string - Provider oauth.ProviderType - Scopes string -} - -func (s *TestkubeAPI) WithFeatureFlags(ff featureflags.FeatureFlags) *TestkubeAPI { - s.featureFlags = ff - return s -} - -type LabelSource interface { - ListLabels() (map[string][]string, error) -} +func (s *TestkubeAPI) Init(server server.HTTPServer) { + // TODO: Consider extracting outside? + server.Routes.Get("/info", s.InfoHandler()) + server.Routes.Get("/debug", s.DebugHandler()) -func (s *TestkubeAPI) WithLabelSources(l ...LabelSource) { - *s.LabelSources = append(*s.LabelSources, l...) -} - -// SendTelemetryStartEvent sends anonymous start event to telemetry trackers -func (s TestkubeAPI) SendTelemetryStartEvent(ctx context.Context, ch chan struct{}) { - go func() { - defer func() { - ch <- struct{}{} - }() - - telemetryEnabled, err := s.ConfigMap.GetTelemetryEnabled(ctx) - if err != nil { - s.Log.Errorw("error getting config map", "error", err) - } - - if !telemetryEnabled { - return - } - - out, err := telemetry.SendServerStartEvent(s.Config.ClusterID, version.Version) - if err != nil { - s.Log.Debug("telemetry send error", "error", err.Error()) - } else { - s.Log.Debugw("sending telemetry server start event", "output", out) - } - }() -} - -func (s *TestkubeAPI) Init(cdEventsTarget string, enableK8sEvents bool) { - s.InitEventListeners( - s.proContext, - s.WebhooksClient, - s.TemplatesClient, - s.ExecutionResults, - s.TestExecutionResults, - s.TestWorkflowResults, - s.Metrics, - cdEventsTarget, - s.Config.ClusterID, - s.Namespace, - s.dashboardURI, - enableK8sEvents, - s.Clientset, - ) - s.InitEnvs() - s.InitRoutes() - s.InitEvents() -} - -// InitEnvs initializes api server settings -func (s *TestkubeAPI) InitEnvs() { - if err := envconfig.Process("STORAGE", &s.storageParams); err != nil { - s.Log.Debugw("Processing STORAGE environment config", err) - } - - if err := envconfig.Process("TESTKUBE_OAUTH", &s.oauthParams); err != nil { - s.Log.Debugw("Processing TESTKUBE_OAUTH environment config", err) - } -} - -func (s *TestkubeAPI) InitRoutes() { - s.Routes.Static("/api-docs", "./api/v1") - s.Routes.Use(cors.New()) - s.Routes.Use(s.AuthHandler()) - - s.Routes.Get("/info", s.InfoHandler()) - s.Routes.Get("/routes", s.RoutesHandler()) - s.Routes.Get("/debug", s.DebugHandler()) - - root := s.Routes - - executors := root.Group("/executors") - - executors.Post("/", s.CreateExecutorHandler()) - executors.Get("/", s.ListExecutorsHandler()) - executors.Get("/:name", s.GetExecutorHandler()) - executors.Patch("/:name", s.UpdateExecutorHandler()) - executors.Delete("/:name", s.DeleteExecutorHandler()) - executors.Delete("/", s.DeleteExecutorsHandler()) - - executorByTypes := root.Group("/executor-by-types") - executorByTypes.Get("/", s.GetExecutorByTestTypeHandler()) + root := server.Routes webhooks := root.Group("/webhooks") @@ -339,73 +127,6 @@ func (s *TestkubeAPI) InitRoutes() { webhooks.Delete("/:name", s.DeleteWebhookHandler()) webhooks.Delete("/", s.DeleteWebhooksHandler()) - executions := root.Group("/executions") - - executions.Get("/", s.ListExecutionsHandler()) - executions.Post("/", s.ExecuteTestsHandler()) - executions.Get("/:executionID", s.GetExecutionHandler()) - executions.Get("/:executionID/artifacts", s.ListArtifactsHandler()) - executions.Get("/:executionID/logs", s.ExecutionLogsHandler()) - executions.Get("/:executionID/logs/stream", s.ExecutionLogsStreamHandler()) - executions.Get("/:executionID/logs/v2", s.ExecutionLogsHandlerV2()) - executions.Get("/:executionID/logs/stream/v2", s.ExecutionLogsStreamHandlerV2()) - executions.Get("/:executionID/artifacts/:filename", s.GetArtifactHandler()) - executions.Get("/:executionID/artifact-archive", s.GetArtifactArchiveHandler()) - - tests := root.Group("/tests") - - tests.Get("/", s.ListTestsHandler()) - tests.Post("/", s.CreateTestHandler()) - tests.Patch("/:id", s.UpdateTestHandler()) - tests.Delete("/", s.DeleteTestsHandler()) - - tests.Get("/:id", s.GetTestHandler()) - tests.Delete("/:id", s.DeleteTestHandler()) - tests.Post("/:id/abort", s.AbortTestHandler()) - - tests.Get("/:id/metrics", s.TestMetricsHandler()) - - tests.Post("/:id/executions", s.ExecuteTestsHandler()) - - tests.Get("/:id/executions", s.ListExecutionsHandler()) - tests.Get("/:id/executions/:executionID", s.GetExecutionHandler()) - tests.Patch("/:id/executions/:executionID", s.AbortExecutionHandler()) - - testWithExecutions := s.Routes.Group("/test-with-executions") - testWithExecutions.Get("/", s.ListTestWithExecutionsHandler()) - testWithExecutions.Get("/:id", s.GetTestWithExecutionHandler()) - - testsuites := root.Group("/test-suites") - - testsuites.Post("/", s.CreateTestSuiteHandler()) - testsuites.Patch("/:id", s.UpdateTestSuiteHandler()) - testsuites.Get("/", s.ListTestSuitesHandler()) - testsuites.Delete("/", s.DeleteTestSuitesHandler()) - testsuites.Get("/:id", s.GetTestSuiteHandler()) - testsuites.Delete("/:id", s.DeleteTestSuiteHandler()) - testsuites.Post("/:id/abort", s.AbortTestSuiteHandler()) - - testsuites.Post("/:id/executions", s.ExecuteTestSuitesHandler()) - testsuites.Get("/:id/executions", s.ListTestSuiteExecutionsHandler()) - testsuites.Get("/:id/executions/:executionID", s.GetTestSuiteExecutionHandler()) - testsuites.Get("/:id/executions/:executionID/artifacts", s.ListTestSuiteArtifactsHandler()) - testsuites.Patch("/:id/executions/:executionID", s.AbortTestSuiteExecutionHandler()) - - testsuites.Get("/:id/tests", s.ListTestSuiteTestsHandler()) - - testsuites.Get("/:id/metrics", s.TestSuiteMetricsHandler()) - - testSuiteExecutions := root.Group("/test-suite-executions") - testSuiteExecutions.Get("/", s.ListTestSuiteExecutionsHandler()) - testSuiteExecutions.Post("/", s.ExecuteTestSuitesHandler()) - testSuiteExecutions.Get("/:executionID", s.GetTestSuiteExecutionHandler()) - testSuiteExecutions.Get("/:executionID/artifacts", s.ListTestSuiteArtifactsHandler()) - testSuiteExecutions.Patch("/:executionID", s.AbortTestSuiteExecutionHandler()) - - testSuiteWithExecutions := root.Group("/test-suite-with-executions") - testSuiteWithExecutions.Get("/", s.ListTestSuiteWithExecutionsHandler()) - testSuiteWithExecutions.Get("/:id", s.GetTestSuiteWithExecutionHandler()) - testWorkflows := root.Group("/test-workflows") testWorkflows.Get("/", s.ListTestWorkflowsHandler()) testWorkflows.Post("/", s.CreateTestWorkflowHandler()) @@ -464,33 +185,12 @@ func (s *TestkubeAPI) InitRoutes() { keymap := root.Group("/keymap") keymap.Get("/triggers", s.GetTestTriggerKeyMapHandler()) - testsources := root.Group("/test-sources") - testsources.Post("/", s.CreateTestSourceHandler()) - testsources.Get("/", s.ListTestSourcesHandler()) - testsources.Patch("/", s.ProcessTestSourceBatchHandler()) - testsources.Get("/:name", s.GetTestSourceHandler()) - testsources.Patch("/:name", s.UpdateTestSourceHandler()) - testsources.Delete("/:name", s.DeleteTestSourceHandler()) - testsources.Delete("/", s.DeleteTestSourcesHandler()) - - templates := root.Group("/templates") - - templates.Post("/", s.CreateTemplateHandler()) - templates.Patch("/:name", s.UpdateTemplateHandler()) - templates.Get("/", s.ListTemplatesHandler()) - templates.Get("/:name", s.GetTemplateHandler()) - templates.Delete("/:name", s.DeleteTemplateHandler()) - templates.Delete("/", s.DeleteTemplatesHandler()) - labels := root.Group("/labels") labels.Get("/", s.ListLabelsHandler()) tags := root.Group("/tags") tags.Get("/", s.ListTagsHandler()) - slack := root.Group("/slack") - slack.Get("/", s.OauthHandler()) - events := root.Group("/events") events.Post("/flux", s.FluxEventHandler()) events.Get("/stream", s.EventsStreamHandler()) @@ -502,12 +202,6 @@ func (s *TestkubeAPI) InitRoutes() { debug := root.Group("/debug") debug.Get("/listeners", s.GetDebugListenersHandler()) - files := root.Group("/uploads") - files.Post("/", s.UploadFiles()) - - // Register TestWorkflows as additional source for labels - s.WithLabelSources(s.TestWorkflowsClient, s.TestWorkflowTemplatesClient) - secrets := root.Group("/secrets") secrets.Get("/", s.ListSecretsHandler()) secrets.Post("/", s.CreateSecretHandler()) @@ -517,217 +211,4 @@ func (s *TestkubeAPI) InitRoutes() { repositories := root.Group("/repositories") repositories.Post("/", s.ValidateRepositoryHandler()) - - // mount dashboard on /ui - dashboardURI := os.Getenv("TESTKUBE_DASHBOARD_URI") - if dashboardURI == "" { - dashboardURI = "http://testkube-dashboard" - } - s.Log.Infow("dashboard uri", "uri", dashboardURI) - s.Mux.All("/", proxy.Forward(dashboardURI)) - - // set up proxy for the internal GraphQL server - s.Mux.All("/graphql", func(c *fiber.Ctx) error { - // Connect to server - serverConn, err := net.Dial("tcp", ":"+s.graphqlPort) - if err != nil { - s.Log.Errorw("could not connect to GraphQL server as a proxy", "error", err) - return err - } - - // Resend headers to the server - _, err = serverConn.Write(c.Request().Header.Header()) - if err != nil { - serverConn.Close() - s.Log.Errorw("error while sending headers to GraphQL server", "error", err) - return err - } - - // Resend body to the server - _, err = serverConn.Write(c.Body()) - if err != nil && err != io.EOF { - serverConn.Close() - s.Log.Errorw("error while reading GraphQL client data", "error", err) - return err - } - - // Handle optional WebSocket connection - c.Context().HijackSetNoResponse(true) - c.Context().Hijack(func(clientConn net.Conn) { - // Close the connection afterward - defer serverConn.Close() - defer clientConn.Close() - - // Extract Unix connection - serverSock, ok := serverConn.(*net.TCPConn) - if !ok { - s.Log.Errorw("error while building TCPConn out ouf serverConn", "error", err) - return - } - clientSock, ok := reflect.Indirect(reflect.ValueOf(clientConn)).FieldByName("Conn").Interface().(*net.TCPConn) - if !ok { - s.Log.Errorw("error while building TCPConn out of hijacked connection", "error", err) - return - } - - // Duplex communication between client and GraphQL server - var wg sync.WaitGroup - wg.Add(2) - go func() { - defer wg.Done() - _, err := io.Copy(clientSock, serverSock) - if err != nil && err != io.EOF && !errors.Is(err, syscall.ECONNRESET) && !errors.Is(err, syscall.EPIPE) { - s.Log.Errorw("error while reading GraphQL client data", "error", err) - } - serverSock.CloseWrite() - }() - go func() { - defer wg.Done() - _, err = io.Copy(serverSock, clientSock) - if err != nil && err != io.EOF { - s.Log.Errorw("error while reading GraphQL server data", "error", err) - } - clientSock.CloseWrite() - }() - wg.Wait() - }) - return nil - }) -} - -func (s *TestkubeAPI) InitEventListeners( - proContext *config.ProContext, - webhookClient *executorsclientv1.WebhooksClient, - templatesClient *templatesclientv1.TemplatesClient, - testExecutionResults result.Repository, - testSuiteExecutionsResults testresult.Repository, - testWorkflowResults testworkflow.Repository, - metrics metrics.Metrics, - cdeventsTarget string, - clusterId string, - namespace string, - dashboardURI string, - enableK8sEvents bool, - clientset kubernetes.Interface, -) { - // will be reused in websockets handler - s.WebsocketLoader = ws.NewWebsocketLoader() - - s.Events.Loader.Register(webhook.NewWebhookLoader( - s.Log, webhookClient, templatesClient, testExecutionResults, testSuiteExecutionsResults, - testWorkflowResults, metrics, s.proContext, s.Envs)) - s.Events.Loader.Register(s.WebsocketLoader) - s.Events.Loader.Register(s.slackLoader) - - if cdeventsTarget != "" { - cdeventLoader, err := cdevent.NewCDEventLoader(cdeventsTarget, clusterId, namespace, dashboardURI, testkube.AllEventTypes) - if err == nil { - s.Events.Loader.Register(cdeventLoader) - } else { - s.Log.Debug("cdevents init error", "error", err.Error()) - } - } - - if enableK8sEvents { - s.Events.Loader.Register(k8sevent.NewK8sEventLoader(clientset, namespace, testkube.AllEventTypes)) - } -} - -func (s TestkubeAPI) StartTelemetryHeartbeats(ctx context.Context, ch chan struct{}) { - go func() { - <-ch - - ticker := time.NewTicker(HeartbeatInterval) - for { - telemetryEnabled, err := s.ConfigMap.GetTelemetryEnabled(ctx) - if err != nil { - s.Log.Errorw("error getting config map", "error", err) - } - if telemetryEnabled { - l := s.Log.With("measurmentId", telemetry.TestkubeMeasurementID, "secret", text.Obfuscate(telemetry.TestkubeMeasurementSecret)) - host, err := os.Hostname() - if err != nil { - l.Debugw("getting hostname error", "hostname", host, "error", err) - } - out, err := telemetry.SendHeartbeatEvent(host, version.Version, s.Config.ClusterID) - if err != nil { - l.Debugw("sending heartbeat telemetry event error", "error", err) - } else { - l.Debugw("sending heartbeat telemetry event", "output", out) - } - - } - <-ticker.C - } - }() -} - -// TODO should we use single generic filter for all list based resources ? -// currently filters for e.g. tests are done "by hand" -func getFilterFromRequest(c *fiber.Ctx) result.Filter { - - filter := result.NewExecutionsFilter() - - // id for /tests/ID/executions - testName := c.Params("id", "") - if testName == "" { - // query param for /executions?testName - testName = c.Query("testName", "") - } - - if testName != "" { - filter = filter.WithTestName(testName) - } - - textSearch := c.Query("textSearch", "") - if textSearch != "" { - filter = filter.WithTextSearch(textSearch) - } - - page, err := strconv.Atoi(c.Query("page", "")) - if err == nil { - filter = filter.WithPage(page) - } - - pageSize, err := strconv.Atoi(c.Query("pageSize", "")) - if err == nil && pageSize != 0 { - filter = filter.WithPageSize(pageSize) - } - - status := c.Query("status", "") - if status != "" { - filter = filter.WithStatus(status) - } - - objectType := c.Query("type", "") - if objectType != "" { - filter = filter.WithType(objectType) - } - - last, err := strconv.Atoi(c.Query("last", "0")) - if err == nil && last != 0 { - filter = filter.WithLastNDays(last) - } - - dFilter := datefilter.NewDateFilter(c.Query("startDate", ""), c.Query("endDate", "")) - if dFilter.IsStartValid { - filter = filter.WithStartDate(dFilter.Start) - } - - if dFilter.IsEndValid { - filter = filter.WithEndDate(dFilter.End) - } - - selector := c.Query("selector") - if selector != "" { - filter = filter.WithSelector(selector) - } - - return filter -} - -// WithProContext sets pro context for the API -func (s *TestkubeAPI) WithProContext(proContext *config.ProContext) *TestkubeAPI { - s.proContext = proContext - return s } diff --git a/internal/app/api/v1/testtriggers.go b/internal/app/api/v1/testtriggers.go index 5434491499a..e1aa07490e8 100644 --- a/internal/app/api/v1/testtriggers.go +++ b/internal/app/api/v1/testtriggers.go @@ -8,11 +8,11 @@ import ( "github.com/gofiber/fiber/v2" k8serrors "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/yaml" testtriggersv1 "github.com/kubeshop/testkube-operator/api/testtriggers/v1" + "github.com/kubeshop/testkube/internal/app/api/apiutils" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/crd" "github.com/kubeshop/testkube/pkg/keymap/triggers" @@ -53,7 +53,7 @@ func (s *TestkubeAPI) CreateTestTriggerHandler() fiber.Handler { if c.Accepts(mediaTypeJSON, mediaTypeYAML) == mediaTypeYAML { data, err := crd.GenerateYAML(crd.TemplateTestTrigger, []testkube.TestTrigger{testtriggersmapper.MapCRDToAPI(&testTrigger)}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } } @@ -61,7 +61,7 @@ func (s *TestkubeAPI) CreateTestTriggerHandler() fiber.Handler { s.Log.Infow("creating test trigger", "testTrigger", testTrigger) - created, err := s.TestKubeClientset.TestsV1().TestTriggers(s.Namespace).Create(c.UserContext(), &testTrigger, v1.CreateOptions{}) + created, err := s.TestTriggersClient.Create(&testTrigger) s.Metrics.IncCreateTestTrigger(err) @@ -102,7 +102,7 @@ func (s *TestkubeAPI) UpdateTestTriggerHandler() fiber.Handler { errPrefix = errPrefix + " " + request.Name // we need to get resource first and load its metadata.ResourceVersion - testTrigger, err := s.TestKubeClientset.TestsV1().TestTriggers(namespace).Get(c.UserContext(), request.Name, v1.GetOptions{}) + testTrigger, err := s.TestTriggersClient.Get(request.Name, namespace) if err != nil { if k8serrors.IsNotFound(err) { return s.Error(c, http.StatusNotFound, fmt.Errorf("%s: client could not find test trigger: %w", errPrefix, err)) @@ -112,10 +112,11 @@ func (s *TestkubeAPI) UpdateTestTriggerHandler() fiber.Handler { // map TestSuite but load spec only to not override metadata.ResourceVersion crdTestTrigger := testtriggersmapper.MapTestTriggerUpsertRequestToTestTriggerCRD(request) + testTrigger.Namespace = namespace testTrigger.Spec = crdTestTrigger.Spec testTrigger.Labels = request.Labels testTrigger.Annotations = request.Annotations - testTrigger, err = s.TestKubeClientset.TestsV1().TestTriggers(namespace).Update(c.UserContext(), testTrigger, v1.UpdateOptions{}) + testTrigger, err = s.TestTriggersClient.Update(testTrigger) s.Metrics.IncUpdateTestTrigger(err) @@ -149,10 +150,7 @@ func (s *TestkubeAPI) BulkUpdateTestTriggersHandler() fiber.Handler { } for namespace := range namespaces { - err = s.TestKubeClientset. - TestsV1(). - TestTriggers(namespace). - DeleteCollection(c.UserContext(), v1.DeleteOptions{}, v1.ListOptions{}) + err = s.TestTriggersClient.DeleteAll(namespace) if err != nil && !k8serrors.IsNotFound(err) { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: error cleaning triggers before reapply", errPrefix)) } @@ -163,20 +161,13 @@ func (s *TestkubeAPI) BulkUpdateTestTriggersHandler() fiber.Handler { testTriggers := make([]testkube.TestTrigger, 0, len(request)) for _, upsertRequest := range request { - namespace := s.Namespace - if upsertRequest.Namespace != "" { - namespace = upsertRequest.Namespace - } var testTrigger *testtriggersv1.TestTrigger crdTestTrigger := testtriggersmapper.MapTestTriggerUpsertRequestToTestTriggerCRD(upsertRequest) // default trigger name if not defined in upsert request if crdTestTrigger.Name == "" { crdTestTrigger.Name = generateTestTriggerName(&crdTestTrigger) } - testTrigger, err = s.TestKubeClientset. - TestsV1(). - TestTriggers(namespace). - Create(c.UserContext(), &crdTestTrigger, v1.CreateOptions{}) + testTrigger, err = s.TestTriggersClient.Create(&crdTestTrigger) s.Metrics.IncCreateTestTrigger(err) @@ -200,7 +191,7 @@ func (s *TestkubeAPI) GetTestTriggerHandler() fiber.Handler { name := c.Params("id") errPrefix := fmt.Sprintf("failed to get test trigger %s", name) - testTrigger, err := s.TestKubeClientset.TestsV1().TestTriggers(namespace).Get(c.UserContext(), name, v1.GetOptions{}) + testTrigger, err := s.TestTriggersClient.Get(name, namespace) if err != nil { if k8serrors.IsNotFound(err) { return s.Warn(c, http.StatusNotFound, fmt.Errorf("%s: client could not find test trigger: %w", errPrefix, err)) @@ -215,7 +206,7 @@ func (s *TestkubeAPI) GetTestTriggerHandler() fiber.Handler { if c.Accepts(mediaTypeJSON, mediaTypeYAML) == mediaTypeYAML { data, err := crd.GenerateYAML(crd.TemplateTestTrigger, []testkube.TestTrigger{apiTestTrigger}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } return c.JSON(apiTestTrigger) @@ -229,7 +220,7 @@ func (s *TestkubeAPI) DeleteTestTriggerHandler() fiber.Handler { name := c.Params("id") errPrefix := fmt.Sprintf("failed to delete test trigger %s", name) - err := s.TestKubeClientset.TestsV1().TestTriggers(namespace).Delete(c.UserContext(), name, v1.DeleteOptions{}) + err := s.TestTriggersClient.Delete(name, namespace) s.Metrics.IncDeleteTestTrigger(err) @@ -258,8 +249,7 @@ func (s *TestkubeAPI) DeleteTestTriggersHandler() fiber.Handler { return s.Error(c, http.StatusBadRequest, fmt.Errorf("%s: error validating selector: %w", errPrefix, err)) } } - listOpts := v1.ListOptions{LabelSelector: selector} - err := s.TestKubeClientset.TestsV1().TestTriggers(namespace).DeleteCollection(c.UserContext(), v1.DeleteOptions{}, listOpts) + err := s.TestTriggersClient.DeleteByLabels(selector, namespace) s.Metrics.IncBulkDeleteTestTrigger(err) @@ -288,8 +278,7 @@ func (s *TestkubeAPI) ListTestTriggersHandler() fiber.Handler { return s.Error(c, http.StatusBadRequest, fmt.Errorf("%s: error validating selector: %w", errPrefix, err)) } } - opts := v1.ListOptions{LabelSelector: selector} - testTriggers, err := s.TestKubeClientset.TestsV1().TestTriggers(namespace).List(c.UserContext(), opts) + testTriggers, err := s.TestTriggersClient.List(selector, namespace) if err != nil { return s.Error(c, http.StatusBadGateway, fmt.Errorf("%s: client could not list test triggers: %w", errPrefix, err)) } @@ -300,7 +289,7 @@ func (s *TestkubeAPI) ListTestTriggersHandler() fiber.Handler { if c.Accepts(mediaTypeJSON, mediaTypeYAML) == mediaTypeYAML { data, err := crd.GenerateYAML(crd.TemplateTestTrigger, apiTestTriggers) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } return c.JSON(apiTestTriggers) diff --git a/internal/app/api/v1/testworkflowexecutions.go b/internal/app/api/v1/testworkflowexecutions.go index c8f9e5bf575..d2272c8117e 100644 --- a/internal/app/api/v1/testworkflowexecutions.go +++ b/internal/app/api/v1/testworkflowexecutions.go @@ -15,6 +15,7 @@ import ( "github.com/gofiber/websocket/v2" "github.com/pkg/errors" + "github.com/kubeshop/testkube/internal/app/api/apiutils" "github.com/kubeshop/testkube/internal/common" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/datefilter" @@ -345,7 +346,7 @@ func (s *TestkubeAPI) AbortAllTestWorkflowExecutionsHandler() fiber.Handler { filter := testworkflow2.NewExecutionsFilter().WithName(name).WithStatus(string(testkube.RUNNING_TestWorkflowStatus)) executions, err := s.TestWorkflowResults.GetExecutions(ctx, filter) if err != nil { - if IsNotFound(err) { + if apiutils.IsNotFound(err) { c.Status(http.StatusNoContent) return nil } @@ -444,29 +445,6 @@ func (s *TestkubeAPI) GetTestWorkflowArtifactArchiveHandler() fiber.Handler { } } -func (s *TestkubeAPI) GetTestWorkflowNotificationsStream(ctx context.Context, executionID string) (<-chan testkube.TestWorkflowExecutionNotification, error) { - // Load the execution - execution, err := s.TestWorkflowResults.Get(ctx, executionID) - if err != nil { - return nil, err - } - - // Start streaming the notifications - notifications := s.ExecutionWorkerClient.Notifications(ctx, execution.Id, executionworkertypes.NotificationsOptions{ - Hints: executionworkertypes.Hints{ - Namespace: execution.Namespace, - Signature: execution.Signature, - ScheduledAt: common.Ptr(execution.ScheduledAt), - }, - }) - - // Pass them down - if notifications.Err() != nil { - return nil, notifications.Err() - } - return notifications.Channel(), nil -} - func getWorkflowExecutionsFilterFromRequest(c *fiber.Ctx) testworkflow2.Filter { filter := testworkflow2.NewExecutionsFilter() name := c.Params("id", "") diff --git a/internal/app/api/v1/testworkflowwithexecutions.go b/internal/app/api/v1/testworkflowwithexecutions.go index 404259a7707..2c2d3d691be 100644 --- a/internal/app/api/v1/testworkflowwithexecutions.go +++ b/internal/app/api/v1/testworkflowwithexecutions.go @@ -9,6 +9,7 @@ import ( "github.com/gofiber/fiber/v2" + "github.com/kubeshop/testkube/internal/app/api/apiutils" "github.com/kubeshop/testkube/pkg/api/v1/testkube" testworkflowmappers "github.com/kubeshop/testkube/pkg/mapper/testworkflows" "github.com/kubeshop/testkube/pkg/repository/result" @@ -30,7 +31,7 @@ func (s *TestkubeAPI) GetTestWorkflowWithExecutionHandler() fiber.Handler { ctx := c.Context() execution, err := s.TestWorkflowResults.GetLatestByTestWorkflow(ctx, name) - if err != nil && !IsNotFound(err) { + if err != nil && !apiutils.IsNotFound(err) { return s.ClientError(c, errPrefix, err) } diff --git a/internal/app/api/v1/utils.go b/internal/app/api/v1/utils.go index 0976985f565..f40692cf2a7 100644 --- a/internal/app/api/v1/utils.go +++ b/internal/app/api/v1/utils.go @@ -2,15 +2,9 @@ package v1 import ( "github.com/gofiber/fiber/v2" - "github.com/pkg/errors" - "go.mongodb.org/mongo-driver/mongo" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/kubeshop/testkube/internal/common" - "github.com/kubeshop/testkube/pkg/secretmanager" ) func ExpectsYAML(c *fiber.Ctx) bool { @@ -54,16 +48,3 @@ func SendCRDs[T interface{}](c *fiber.Ctx, kind string, groupVersion schema.Grou c.Context().SetContentType(mediaTypeYAML) return c.Send(b) } - -func IsNotFound(err error) bool { - if err == nil { - return false - } - if errors.Is(err, mongo.ErrNoDocuments) || k8serrors.IsNotFound(err) || errors.Is(err, secretmanager.ErrNotFound) { - return true - } - if e, ok := status.FromError(err); ok { - return e.Code() == codes.NotFound - } - return false -} diff --git a/internal/app/api/v1/webhook.go b/internal/app/api/v1/webhook.go index c9c786bb1c0..8bcdec7281c 100644 --- a/internal/app/api/v1/webhook.go +++ b/internal/app/api/v1/webhook.go @@ -10,12 +10,13 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" executorv1 "github.com/kubeshop/testkube-operator/api/executor/v1" + "github.com/kubeshop/testkube/internal/app/api/apiutils" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/crd" webhooksmapper "github.com/kubeshop/testkube/pkg/mapper/webhooks" ) -func (s TestkubeAPI) CreateWebhookHandler() fiber.Handler { +func (s *TestkubeAPI) CreateWebhookHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to create webhook" var webhook executorv1.Webhook @@ -38,7 +39,7 @@ func (s TestkubeAPI) CreateWebhookHandler() fiber.Handler { } data, err := crd.GenerateYAML(crd.TemplateWebhook, []testkube.WebhookCreateRequest{request}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } webhook = webhooksmapper.MapAPIToCRD(request) @@ -55,7 +56,7 @@ func (s TestkubeAPI) CreateWebhookHandler() fiber.Handler { } } -func (s TestkubeAPI) UpdateWebhookHandler() fiber.Handler { +func (s *TestkubeAPI) UpdateWebhookHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to update webhook" var request testkube.WebhookUpdateRequest @@ -102,7 +103,7 @@ func (s TestkubeAPI) UpdateWebhookHandler() fiber.Handler { } } -func (s TestkubeAPI) ListWebhooksHandler() fiber.Handler { +func (s *TestkubeAPI) ListWebhooksHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to list webhooks" @@ -125,14 +126,14 @@ func (s TestkubeAPI) ListWebhooksHandler() fiber.Handler { } data, err := crd.GenerateYAML(crd.TemplateWebhook, results) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } return c.JSON(results) } } -func (s TestkubeAPI) GetWebhookHandler() fiber.Handler { +func (s *TestkubeAPI) GetWebhookHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("name") errPrefix := fmt.Sprintf("failed to get webhook %s", name) @@ -152,14 +153,14 @@ func (s TestkubeAPI) GetWebhookHandler() fiber.Handler { } data, err := crd.GenerateYAML(crd.TemplateWebhook, []testkube.Webhook{result}) - return s.getCRDs(c, data, err) + return apiutils.SendLegacyCRDs(c, data, err) } return c.JSON(result) } } -func (s TestkubeAPI) DeleteWebhookHandler() fiber.Handler { +func (s *TestkubeAPI) DeleteWebhookHandler() fiber.Handler { return func(c *fiber.Ctx) error { name := c.Params("name") errPrefix := fmt.Sprintf("failed to delete webhook %s", name) @@ -177,7 +178,7 @@ func (s TestkubeAPI) DeleteWebhookHandler() fiber.Handler { } } -func (s TestkubeAPI) DeleteWebhooksHandler() fiber.Handler { +func (s *TestkubeAPI) DeleteWebhooksHandler() fiber.Handler { return func(c *fiber.Ctx) error { errPrefix := "failed to delete webhooks" diff --git a/internal/config/config.go b/internal/config/config.go index 7451e6d5be9..d763de318a9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,7 +7,7 @@ import ( ) type Config struct { - APIServerPort string `envconfig:"APISERVER_PORT" default:"8088"` + APIServerPort int `envconfig:"APISERVER_PORT" default:"8088"` APIServerConfig string `envconfig:"APISERVER_CONFIG" default:""` APIServerFullname string `envconfig:"APISERVER_FULLNAME" default:"testkube-api-server"` APIMongoDSN string `envconfig:"API_MONGO_DSN" default:"mongodb://localhost:27017"` @@ -88,7 +88,7 @@ type Config struct { // TestkubeImageCredentialsCacheTTL is the duration for which the image pull credentials should be cached provided as a Go duration string. // If set to 0, the cache is disabled. TestkubeImageCredentialsCacheTTL time.Duration `envconfig:"TESTKUBE_IMAGE_CREDENTIALS_CACHE_TTL" default:"30m"` - GraphqlPort string `envconfig:"TESTKUBE_GRAPHQL_PORT" default:"8070"` + GraphqlPort int `envconfig:"TESTKUBE_GRAPHQL_PORT" default:"8070"` CDEventsTarget string `envconfig:"CDEVENTS_TARGET" default:""` TestkubeDashboardURI string `envconfig:"TESTKUBE_DASHBOARD_URI" default:""` DisableReconciler bool `envconfig:"DISABLE_RECONCILER" default:"false"` diff --git a/internal/graphql/server.go b/internal/graphql/server.go index 1ce3fbeb18c..159dda412e9 100644 --- a/internal/graphql/server.go +++ b/internal/graphql/server.go @@ -17,7 +17,7 @@ import ( "github.com/kubeshop/testkube/pkg/log" ) -func GetServer(eventBus bus.Bus, executorsClient *executorsclientv1.ExecutorsClient) *handler.Server { +func GetServer(eventBus bus.Bus, executorsClient executorsclientv1.Interface) *handler.Server { service := services.NewService(eventBus, log.DefaultLogger) resolver := &resolvers.Resolver{ ExecutorsService: services.NewExecutorsService(service, executorsClient), diff --git a/internal/graphql/services/executors.go b/internal/graphql/services/executors.go index a0b25318021..e2254cddf2a 100644 --- a/internal/graphql/services/executors.go +++ b/internal/graphql/services/executors.go @@ -16,10 +16,10 @@ type ExecutorsService interface { type executorsService struct { ServiceBase - client *executorsclientv1.ExecutorsClient + client executorsclientv1.Interface } -func NewExecutorsService(service Service, client *executorsclientv1.ExecutorsClient) ExecutorsService { +func NewExecutorsService(service Service, client executorsclientv1.Interface) ExecutorsService { return &executorsService{ServiceBase: ServiceBase{Service: service}, client: client} } diff --git a/internal/migrations/init.go b/internal/migrations/init.go deleted file mode 100644 index 35c2b9a2244..00000000000 --- a/internal/migrations/init.go +++ /dev/null @@ -1,9 +0,0 @@ -package migrations - -import "github.com/kubeshop/testkube/pkg/migrator" - -var Migrator migrator.Migrator - -func init() { - Migrator = *migrator.NewMigrator() -} diff --git a/internal/migrations/mongo_statuses_migration.js b/internal/migrations/mongo_statuses_migration.js deleted file mode 100644 index 22adffd9d79..00000000000 --- a/internal/migrations/mongo_statuses_migration.js +++ /dev/null @@ -1,8 +0,0 @@ -db.results.find( {'executionresult.status':'success'}, {'executionresult.status':1} ).forEach( function(r) { return db.results.update({_id: r._id}, {$set: {'executionresult.status':'passed' }}) } ) -db.results.find( {'executionresult.status':'pending'}, {'executionresult.status':1} ).forEach( function(r) { return db.results.update({_id: r._id}, {$set: {'executionresult.status':'running' }}) } ) -db.results.find( {'executionresult.status':'error'}, {'executionresult.status':1} ).forEach( function(r) { return db.results.update({_id: r._id}, {$set: {'executionresult.status':'failed' }}) } ) - - -db.testresults.find( {'status':'success'}, {'status':1} ).forEach( function(r) { return db.testresults.update({_id: r._id}, {$set: {'status':'passed' }}) } ) -db.testresults.find( {'status':'pending'}, {'status':1} ).forEach( function(r) { return db.testresults.update({_id: r._id}, {$set: {'status':'running' }}) } ) -db.testresults.find( {'status':'error'}, {'status':1} ).forEach( function(r) { return db.testresults.update({_id: r._id}, {$set: {'status':'failed' }}) } ) \ No newline at end of file diff --git a/internal/migrations/version_0.8.8.go b/internal/migrations/version_0.8.8.go deleted file mode 100644 index a0e0ecb3b36..00000000000 --- a/internal/migrations/version_0.8.8.go +++ /dev/null @@ -1,39 +0,0 @@ -package migrations - -import "github.com/kubeshop/testkube/pkg/migrator" - -// add migration to global migrator -func init() { - Migrator.Add(NewVersion_0_8_8()) -} - -func NewVersion_0_8_8() *Version_0_8_8 { - return &Version_0_8_8{} -} - -type Version_0_8_8 struct { -} - -func (m *Version_0_8_8) Version() string { - return "0.8.8" -} -func (m *Version_0_8_8) Migrate() error { - commands := []string{ - `kubectl annotate --overwrite crds executors.executor.testkube.io meta.helm.sh/release-name=testkube meta.helm.sh/release-namespace=testkube`, - `kubectl annotate --overwrite crds tests.tests.testkube.io meta.helm.sh/release-name=testkube meta.helm.sh/release-namespace=testkube`, - `kubectl annotate --overwrite crds scripts.tests.testkube.io meta.helm.sh/release-name=testkube meta.helm.sh/release-namespace=testkube`, - `kubectl label --overwrite crds executors.executor.testkube.io app.kubernetes.io/managed-by=Helm`, - `kubectl label --overwrite crds tests.tests.testkube.io app.kubernetes.io/managed-by=Helm`, - `kubectl label --overwrite crds scripts.tests.testkube.io app.kubernetes.io/managed-by=Helm`, - } - - _, err := Migrator.ExecuteCommands(commands) - return err -} -func (m *Version_0_8_8) Info() string { - return "Adding labels and annotations to Testkube CRDs" -} - -func (m *Version_0_8_8) Type() migrator.MigrationType { - return migrator.MigrationTypeClient -} diff --git a/internal/migrations/version_0.9.2.go b/internal/migrations/version_0.9.2.go deleted file mode 100644 index e3a5041808d..00000000000 --- a/internal/migrations/version_0.9.2.go +++ /dev/null @@ -1,193 +0,0 @@ -package migrations - -import ( - "strings" - - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - testsv1 "github.com/kubeshop/testkube-operator/api/tests/v1" - testsv3 "github.com/kubeshop/testkube-operator/api/tests/v3" - testsuite "github.com/kubeshop/testkube-operator/api/testsuite/v2" - scriptsclientv2 "github.com/kubeshop/testkube-operator/pkg/client/scripts/v2" - testsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/tests" - testsclientv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" - testsuitesclientv2 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v2" - "github.com/kubeshop/testkube/pkg/api/v1/testkube" - "github.com/kubeshop/testkube/pkg/migrator" -) - -func NewVersion_0_9_2( - scriptsClient *scriptsclientv2.ScriptsClient, - testsClientV1 *testsclientv1.TestsClient, - testsClientV3 *testsclientv3.TestsClient, - testsuitesClient *testsuitesclientv2.TestSuitesClient, -) *Version_0_9_2 { - return &Version_0_9_2{ - scriptsClient: scriptsClient, - testsClientV1: testsClientV1, - testsClientV3: testsClientV3, - testsuitesClient: testsuitesClient, - } -} - -type Version_0_9_2 struct { - scriptsClient *scriptsclientv2.ScriptsClient - testsClientV1 *testsclientv1.TestsClient - testsClientV3 *testsclientv3.TestsClient - testsuitesClient *testsuitesclientv2.TestSuitesClient -} - -func (m *Version_0_9_2) Version() string { - return "0.9.2" -} -func (m *Version_0_9_2) Migrate() error { - scripts, err := m.scriptsClient.List(nil) - if err != nil { - return err - } - - for _, script := range scripts.Items { - if _, err = m.testsClientV3.Get(script.Name); err != nil && !errors.IsNotFound(err) { - return err - } - - if err == nil { - continue - } - - test := &testsv3.Test{ - ObjectMeta: metav1.ObjectMeta{ - Name: script.Name, - Namespace: script.Namespace, - }, - Spec: testsv3.TestSpec{ - Type_: script.Spec.Type_, - Name: script.Spec.Name, - }, - } - - if len(script.Spec.Params) != 0 { - test.Spec.ExecutionRequest = &testsv3.ExecutionRequest{ - Variables: make(map[string]testsv3.Variable, len(script.Spec.Params)), - } - - for key, value := range script.Spec.Params { - test.Spec.ExecutionRequest.Variables[key] = testsv3.Variable{ - Name: key, - Value: value, - Type_: string(*testkube.VariableTypeBasic), - } - } - } - - if script.Spec.Content != nil { - test.Spec.Content = &testsv3.TestContent{ - Type_: testsv3.TestContentType(script.Spec.Content.Type_), - Data: script.Spec.Content.Data, - Uri: script.Spec.Content.Uri, - } - - if script.Spec.Content.Repository != nil { - test.Spec.Content.Repository = &testsv3.Repository{ - Type_: script.Spec.Content.Repository.Type_, - Uri: script.Spec.Content.Repository.Uri, - Branch: script.Spec.Content.Repository.Branch, - Path: script.Spec.Content.Repository.Path, - } - } - } - - if _, err = m.testsClientV3.Create(test, false); err != nil { - return err - } - - if err = m.scriptsClient.Delete(script.Name); err != nil { - return err - } - } - - tests, err := m.testsClientV1.List(nil) - if err != nil { - return err - } - -OUTER: - for _, test := range tests.Items { - if _, err = m.testsuitesClient.Get(test.Name); err != nil && !errors.IsNotFound(err) { - return err - } - - if err == nil { - continue - } - - for _, managedField := range test.GetManagedFields() { - if !strings.HasSuffix(managedField.APIVersion, "/v1") { - continue OUTER - } - } - - testsuite := &testsuite.TestSuite{ - ObjectMeta: metav1.ObjectMeta{ - Name: test.Name, - Namespace: test.Namespace, - }, - Spec: testsuite.TestSuiteSpec{ - Repeats: test.Spec.Repeats, - Description: test.Spec.Description, - }, - } - - for _, step := range test.Spec.Before { - testsuite.Spec.Before = append(testsuite.Spec.Before, copyTestStepTest2Testsuite(step)) - } - - for _, step := range test.Spec.Steps { - testsuite.Spec.Steps = append(testsuite.Spec.Steps, copyTestStepTest2Testsuite(step)) - } - - for _, step := range test.Spec.After { - testsuite.Spec.After = append(testsuite.Spec.After, copyTestStepTest2Testsuite(step)) - } - - if _, err = m.testsuitesClient.Create(testsuite); err != nil { - return err - } - - if err = m.testsClientV1.Delete(test.Name); err != nil { - return err - } - } - - return nil -} -func (m *Version_0_9_2) Info() string { - return "Moving scripts v2 resources to tests v2 ones and tests v1 resources to testsuites v1 ones" -} - -func (m *Version_0_9_2) Type() migrator.MigrationType { - return migrator.MigrationTypeServer -} - -func copyTestStepTest2Testsuite(step testsv1.TestStepSpec) testsuite.TestSuiteStepSpec { - result := testsuite.TestSuiteStepSpec{ - Type: testsuite.TestSuiteStepType(step.Type), - } - - if step.Execute != nil { - result.Execute = &testsuite.TestSuiteStepExecute{ - Namespace: step.Execute.Namespace, - Name: step.Execute.Name, - StopOnFailure: step.Execute.StopOnFailure, - } - } - - if step.Delay != nil { - result.Delay = &testsuite.TestSuiteStepDelay{ - Duration: step.Delay.Duration, - } - } - - return result -} diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index c886953afb5..575b618e401 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -153,11 +153,10 @@ type Agent struct { clusterID string clusterName string - envs map[string]string features featureflags.FeatureFlags dockerImageVersion string - proContext config.ProContext + proContext *config.ProContext } func NewAgent(logger *zap.SugaredLogger, @@ -167,9 +166,8 @@ func NewAgent(logger *zap.SugaredLogger, workflowNotificationsFunc func(ctx context.Context, executionID string) (<-chan testkube.TestWorkflowExecutionNotification, error), clusterID string, clusterName string, - envs map[string]string, features featureflags.FeatureFlags, - proContext config.ProContext, + proContext *config.ProContext, dockerImageVersion string, ) (*Agent, error) { return &Agent{ @@ -194,7 +192,6 @@ func NewAgent(logger *zap.SugaredLogger, testWorkflowNotificationsFunc: workflowNotificationsFunc, clusterID: clusterID, clusterName: clusterName, - envs: envs, features: features, proContext: proContext, dockerImageVersion: dockerImageVersion, diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 12306a2076b..8cf5c15688b 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -61,7 +61,7 @@ func TestCommandExecution(t *testing.T) { logger, _ := zap.NewDevelopment() proContext := config.ProContext{APIKey: "api-key", WorkerCount: 5, LogStreamWorkerCount: 5, WorkflowNotificationsWorkerCount: 5} - agent, err := agent.NewAgent(logger.Sugar(), m, grpcClient, logStreamFunc, workflowNotificationsStreamFunc, "", "", nil, featureflags.FeatureFlags{}, proContext, "") + agent, err := agent.NewAgent(logger.Sugar(), m, grpcClient, logStreamFunc, workflowNotificationsStreamFunc, "", "", featureflags.FeatureFlags{}, &proContext, "") if err != nil { t.Fatal(err) } diff --git a/pkg/agent/events_test.go b/pkg/agent/events_test.go index addfcc3918e..eeaab03be9e 100644 --- a/pkg/agent/events_test.go +++ b/pkg/agent/events_test.go @@ -57,7 +57,7 @@ func TestEventLoop(t *testing.T) { var workflowNotificationsStreamFunc func(ctx context.Context, executionID string) (<-chan testkube.TestWorkflowExecutionNotification, error) proContext := config.ProContext{APIKey: "api-key", WorkerCount: 5, LogStreamWorkerCount: 5, WorkflowNotificationsWorkerCount: 5} - agent, err := agent.NewAgent(logger.Sugar(), nil, grpcClient, logStreamFunc, workflowNotificationsStreamFunc, "", "", nil, featureflags.FeatureFlags{}, proContext, "") + agent, err := agent.NewAgent(logger.Sugar(), nil, grpcClient, logStreamFunc, workflowNotificationsStreamFunc, "", "", featureflags.FeatureFlags{}, &proContext, "") assert.NoError(t, err) go func() { l, err := agent.Load() diff --git a/pkg/agent/logs_test.go b/pkg/agent/logs_test.go index 0a1ae888082..11fc9bfc797 100644 --- a/pkg/agent/logs_test.go +++ b/pkg/agent/logs_test.go @@ -68,7 +68,7 @@ func TestLogStream(t *testing.T) { logger, _ := zap.NewDevelopment() proContext := config.ProContext{APIKey: "api-key", WorkerCount: 5, LogStreamWorkerCount: 5, WorkflowNotificationsWorkerCount: 5} - agent, err := agent.NewAgent(logger.Sugar(), m, grpcClient, logStreamFunc, workflowNotificationsStreamFunc, "", "", nil, featureflags.FeatureFlags{}, proContext, "") + agent, err := agent.NewAgent(logger.Sugar(), m, grpcClient, logStreamFunc, workflowNotificationsStreamFunc, "", "", featureflags.FeatureFlags{}, &proContext, "") if err != nil { t.Fatal(err) } diff --git a/pkg/api/v1/testkube/model_debug_info.go b/pkg/api/v1/testkube/model_debug_info.go index bdd2feff5a3..665596106a0 100644 --- a/pkg/api/v1/testkube/model_debug_info.go +++ b/pkg/api/v1/testkube/model_debug_info.go @@ -11,10 +11,9 @@ package testkube // Testkube debug info type DebugInfo struct { - ClientVersion string `json:"clientVersion,omitempty"` - ServerVersion string `json:"serverVersion,omitempty"` - ClusterVersion string `json:"clusterVersion,omitempty"` - ApiLogs []string `json:"apiLogs,omitempty"` - OperatorLogs []string `json:"operatorLogs,omitempty"` - ExecutionLogs map[string][]string `json:"executionLogs,omitempty"` + ClientVersion string `json:"clientVersion,omitempty"` + ServerVersion string `json:"serverVersion,omitempty"` + ClusterVersion string `json:"clusterVersion,omitempty"` + ApiLogs []string `json:"apiLogs,omitempty"` + OperatorLogs []string `json:"operatorLogs,omitempty"` } diff --git a/pkg/cloud/client/rest.go b/pkg/cloud/client/rest.go index 113b9a99b2a..df2d2489f56 100644 --- a/pkg/cloud/client/rest.go +++ b/pkg/cloud/client/rest.go @@ -26,7 +26,8 @@ type RESTClient[T All] struct { } func (c RESTClient[T]) List() ([]T, error) { - r, err := nethttp.NewRequest("GET", c.BaseUrl+c.Path, nil) + path := c.Path + r, err := nethttp.NewRequest("GET", c.BaseUrl+path, nil) r.Header.Add("Authorization", "Bearer "+c.Token) if err != nil { return nil, err @@ -36,6 +37,14 @@ func (c RESTClient[T]) List() ([]T, error) { return nil, err } + if resp.StatusCode >= 400 { + d, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error getting %s: can't read response: %s", c.Path, err) + } + return nil, fmt.Errorf("error getting %s: %s", path, d) + } + var orgsResponse ListResponse[T] err = json.NewDecoder(resp.Body).Decode(&orgsResponse) return orgsResponse.Elements, err @@ -56,7 +65,7 @@ func (c RESTClient[T]) Get(id string) (e T, err error) { if resp.StatusCode > 299 { d, err := io.ReadAll(resp.Body) if err != nil { - return e, fmt.Errorf("error creating %s: can't read response: %s", c.Path, err) + return e, fmt.Errorf("error getting %s: can't read response: %s", c.Path, err) } return e, fmt.Errorf("error getting %s: %s", path, d) } diff --git a/pkg/cloud/data/config/config.go b/pkg/cloud/data/config/config.go index 62ad485e3f2..bc7d62f0d9a 100644 --- a/pkg/cloud/data/config/config.go +++ b/pkg/cloud/data/config/config.go @@ -18,6 +18,7 @@ type CloudRepository struct { executor executor.Executor } +// TODO: Delete, as it's no longer used (may be need to kept for backwards compatibility [?]) func NewCloudResultRepository(cloudClient cloud.TestKubeCloudAPIClient, grpcConn *grpc.ClientConn, apiKey string) *CloudRepository { return &CloudRepository{executor: executor.NewCloudGRPCExecutor(cloudClient, grpcConn, apiKey)} } diff --git a/pkg/configmap/client.go b/pkg/configmap/client.go index 68a432d7db5..7fd2deb38ed 100644 --- a/pkg/configmap/client.go +++ b/pkg/configmap/client.go @@ -23,7 +23,7 @@ type Interface interface { // Client provide methods to manage configmaps type Client struct { - ClientSet *kubernetes.Clientset + ClientSet kubernetes.Interface Log *zap.SugaredLogger Namespace string } @@ -34,12 +34,16 @@ func NewClient(namespace string) (*Client, error) { if err != nil { return nil, err } + return NewClientFor(clientSet, namespace), nil +} +// NewClientFor is a method to create new configmap client using existing clientSet +func NewClientFor(clientSet kubernetes.Interface, namespace string) *Client { return &Client{ ClientSet: clientSet, Log: log.DefaultLogger, Namespace: namespace, - }, nil + } } // Create is a method to create new configmap diff --git a/pkg/cronjob/client.go b/pkg/cronjob/client.go deleted file mode 100644 index d0c67abf001..00000000000 --- a/pkg/cronjob/client.go +++ /dev/null @@ -1,220 +0,0 @@ -package cronjob - -import ( - "bytes" - "context" - "fmt" - "strings" - - "go.uber.org/zap" - v1 "k8s.io/api/batch/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/yaml" - batchv1 "k8s.io/client-go/applyconfigurations/batch/v1" - "k8s.io/client-go/kubernetes" - kyaml "sigs.k8s.io/kustomize/kyaml/yaml" - "sigs.k8s.io/kustomize/kyaml/yaml/merge2" - - "github.com/kubeshop/testkube/pkg/k8sclient" - "github.com/kubeshop/testkube/pkg/log" - "github.com/kubeshop/testkube/pkg/utils" -) - -// Client data struct for managing running cron jobs -type Client struct { - ClientSet *kubernetes.Clientset - Log *zap.SugaredLogger - serviceName string - servicePort int - cronJobTemplate string - Namespace string -} - -type CronJobOptions struct { - Schedule string - Resource string - Data string - Labels map[string]string - CronJobTemplate string - CronJobTemplateExtensions string -} - -type templateParameters struct { - Id string - Name string - Namespace string - ServiceName string - ServicePort int - Schedule string - Resource string - CronJobTemplate string - CronJobTemplateExtensions string - Data string - Labels map[string]string -} - -// NewClient is a method to create new cron job client -func NewClient(serviceName string, servicePort int, cronJobTemplate string, namespace string) (*Client, error) { - clientSet, err := k8sclient.ConnectToK8s() - if err != nil { - return nil, err - } - - return &Client{ - ClientSet: clientSet, - Log: log.DefaultLogger, - serviceName: serviceName, - servicePort: servicePort, - cronJobTemplate: cronJobTemplate, - Namespace: namespace, - }, nil -} - -// Get is a method to retrieve an existing cron job -func (c *Client) Get(name string) (*v1.CronJob, error) { - cronJobClient := c.ClientSet.BatchV1().CronJobs(c.Namespace) - ctx := context.Background() - - cronJob, err := cronJobClient.Get(ctx, name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - - return cronJob, nil -} - -// Apply is a method to create or update a cron job -func (c *Client) Apply(id, name string, options CronJobOptions) error { - template := c.cronJobTemplate - if options.CronJobTemplate != "" { - template = options.CronJobTemplate - } - - cronJobClient := c.ClientSet.BatchV1().CronJobs(c.Namespace) - ctx := context.Background() - - parameters := templateParameters{ - Id: id, - Name: name, - Namespace: c.Namespace, - ServiceName: c.serviceName, - ServicePort: c.servicePort, - Schedule: options.Schedule, - Resource: options.Resource, - CronJobTemplate: template, - CronJobTemplateExtensions: options.CronJobTemplateExtensions, - Data: options.Data, - Labels: options.Labels, - } - - cronJobSpec, err := NewApplySpec(c.Log, parameters) - if err != nil { - return err - } - - if _, err := cronJobClient.Apply(ctx, cronJobSpec, metav1.ApplyOptions{ - FieldManager: "application/apply-patch"}); err != nil { - return err - } - - return nil -} - -// UpdateLabels is a method to update an existing cron job labels -func (c *Client) UpdateLabels(cronJobSpec *v1.CronJob, oldLabels, newLabels map[string]string) error { - cronJobClient := c.ClientSet.BatchV1().CronJobs(c.Namespace) - ctx := context.Background() - - for key := range oldLabels { - delete(cronJobSpec.Labels, key) - } - - for key, value := range newLabels { - cronJobSpec.Labels[key] = value - } - - if _, err := cronJobClient.Update(ctx, cronJobSpec, metav1.UpdateOptions{}); err != nil { - return err - } - - return nil -} - -// Delete is a method to delete an existing cron job -func (c *Client) Delete(name string) error { - cronJobClient := c.ClientSet.BatchV1().CronJobs(c.Namespace) - ctx := context.Background() - - if err := cronJobClient.Delete(ctx, name, metav1.DeleteOptions{}); err != nil { - return err - } - - return nil -} - -// DeleteAll is a method to delete all existing cron jobs -func (c *Client) DeleteAll(resource, selector string) error { - cronJobClient := c.ClientSet.BatchV1().CronJobs(c.Namespace) - ctx := context.Background() - - filter := fmt.Sprintf("testkube=%s", resource) - if selector != "" { - filter += "," + selector - } - - if err := cronJobClient.DeleteCollection(ctx, metav1.DeleteOptions{}, - metav1.ListOptions{LabelSelector: filter}); err != nil { - return err - } - - return nil -} - -// NewApplySpec is a method to return cron job apply spec -func NewApplySpec(log *zap.SugaredLogger, parameters templateParameters) (*batchv1.CronJobApplyConfiguration, error) { - tmpl, err := utils.NewTemplate("cronJob").Parse(parameters.CronJobTemplate) - if err != nil { - return nil, fmt.Errorf("creating cron job spec from options.CronJobTemplate error: %w", err) - } - - parameters.Data = strings.ReplaceAll(parameters.Data, "'", "''''") - var buffer bytes.Buffer - if err = tmpl.ExecuteTemplate(&buffer, "cronJob", parameters); err != nil { - return nil, fmt.Errorf("executing cron job spec template: %w", err) - } - - var cronJob batchv1.CronJobApplyConfiguration - cronJobSpec := buffer.String() - if parameters.CronJobTemplateExtensions != "" { - tmplExt, err := utils.NewTemplate("cronJobExt").Parse(parameters.CronJobTemplateExtensions) - if err != nil { - return nil, fmt.Errorf("creating cron job extensions spec from default template error: %w", err) - } - - var bufferExt bytes.Buffer - if err = tmplExt.ExecuteTemplate(&bufferExt, "cronJobExt", parameters); err != nil { - return nil, fmt.Errorf("executing cron job extensions spec default template: %w", err) - } - - if cronJobSpec, err = merge2.MergeStrings(bufferExt.String(), cronJobSpec, false, kyaml.MergeOptions{}); err != nil { - return nil, fmt.Errorf("merging cron job spec templates: %w", err) - } - } - - log.Debug("Cron job specification", cronJobSpec) - decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewBufferString(cronJobSpec), len(cronJobSpec)) - if err := decoder.Decode(&cronJob); err != nil { - return nil, fmt.Errorf("decoding cron job spec error: %w", err) - } - - for key, value := range parameters.Labels { - cronJob.Labels[key] = value - } - - return &cronJob, nil -} - -// GetMetadataName returns cron job metadata name -func GetMetadataName(name, resource string) string { - return fmt.Sprintf("%s-%s", name, resource) -} diff --git a/pkg/event/kind/webhook/loader.go b/pkg/event/kind/webhook/loader.go index 6e2d490fc90..e2682af54b3 100644 --- a/pkg/event/kind/webhook/loader.go +++ b/pkg/event/kind/webhook/loader.go @@ -5,7 +5,7 @@ import ( "go.uber.org/zap" - executorsv1 "github.com/kubeshop/testkube-operator/api/executor/v1" + executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" templatesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1" v1 "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/internal/config" @@ -19,12 +19,7 @@ import ( var _ common.ListenerLoader = (*WebhooksLoader)(nil) -// WebhooksLister loads webhooks from kubernetes -type WebhooksLister interface { - List(selector string) (*executorsv1.WebhookList, error) -} - -func NewWebhookLoader(log *zap.SugaredLogger, webhooksClient WebhooksLister, templatesClient templatesclientv1.Interface, +func NewWebhookLoader(log *zap.SugaredLogger, webhooksClient executorsclientv1.WebhooksInterface, templatesClient templatesclientv1.Interface, testExecutionResults result.Repository, testSuiteExecutionResults testresult.Repository, testWorkflowExecutionResults testworkflow.Repository, metrics v1.Metrics, proContext *config.ProContext, envs map[string]string, ) *WebhooksLoader { @@ -43,7 +38,7 @@ func NewWebhookLoader(log *zap.SugaredLogger, webhooksClient WebhooksLister, tem type WebhooksLoader struct { log *zap.SugaredLogger - WebhooksClient WebhooksLister + WebhooksClient executorsclientv1.WebhooksInterface templatesClient templatesclientv1.Interface testExecutionResults result.Repository testSuiteExecutionResults testresult.Repository diff --git a/pkg/event/kind/webhook/loader_test.go b/pkg/event/kind/webhook/loader_test.go index 55dbc3e444f..5261091506e 100644 --- a/pkg/event/kind/webhook/loader_test.go +++ b/pkg/event/kind/webhook/loader_test.go @@ -8,21 +8,11 @@ import ( "go.uber.org/zap" executorsv1 "github.com/kubeshop/testkube-operator/api/executor/v1" + executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" templatesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1" v1 "github.com/kubeshop/testkube/internal/app/api/metrics" ) -type DummyLoader struct { -} - -func (l DummyLoader) List(selector string) (*executorsv1.WebhookList, error) { - return &executorsv1.WebhookList{ - Items: []executorsv1.Webhook{ - {Spec: executorsv1.WebhookSpec{Uri: "http://localhost:3333", Events: []executorsv1.EventType{"start-test"}, PayloadObjectField: "text", PayloadTemplate: "{{ .Id }}", Headers: map[string]string{"Content-Type": "application/xml"}}}, - }, - }, nil -} - func TestWebhookLoader(t *testing.T) { t.Parallel() @@ -30,7 +20,14 @@ func TestWebhookLoader(t *testing.T) { defer mockCtrl.Finish() mockTemplatesClient := templatesclientv1.NewMockInterface(mockCtrl) - webhooksLoader := NewWebhookLoader(zap.NewNop().Sugar(), &DummyLoader{}, mockTemplatesClient, nil, nil, nil, v1.NewMetrics(), nil, nil) + mockWebhooksClient := executorsclientv1.NewMockWebhooksInterface(mockCtrl) + mockWebhooksClient.EXPECT().List(gomock.Any()).Return(&executorsv1.WebhookList{ + Items: []executorsv1.Webhook{ + {Spec: executorsv1.WebhookSpec{Uri: "http://localhost:3333", Events: []executorsv1.EventType{"start-test"}, PayloadObjectField: "text", PayloadTemplate: "{{ .Id }}", Headers: map[string]string{"Content-Type": "application/xml"}}}, + }, + }, nil).AnyTimes() + + webhooksLoader := NewWebhookLoader(zap.NewNop().Sugar(), mockWebhooksClient, mockTemplatesClient, nil, nil, nil, v1.NewMetrics(), nil, nil) listeners, err := webhooksLoader.Load() assert.Equal(t, 1, len(listeners)) diff --git a/pkg/executor/client/job.go b/pkg/executor/client/job.go index f1350658a62..04260a8c339 100644 --- a/pkg/executor/client/job.go +++ b/pkg/executor/client/job.go @@ -14,6 +14,7 @@ import ( "text/template" "time" + "github.com/kubeshop/testkube/cmd/api-server/commons" "github.com/kubeshop/testkube/pkg/featureflags" "github.com/kubeshop/testkube/pkg/repository/config" @@ -21,8 +22,6 @@ import ( "github.com/kubeshop/testkube/pkg/version" - "github.com/kubeshop/testkube/pkg/repository/result" - "go.uber.org/zap" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -36,8 +35,6 @@ import ( executorv1 "github.com/kubeshop/testkube-operator/api/executor/v1" templatesv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1" - testexecutionsv1 "github.com/kubeshop/testkube-operator/pkg/client/testexecutions/v1" - testsv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/event" "github.com/kubeshop/testkube/pkg/executor" @@ -79,17 +76,15 @@ const ( // NewJobExecutor creates new job executor func NewJobExecutor( - repo result.Repository, + deprecatedRepositories commons.DeprecatedRepositories, + deprecatedClients commons.DeprecatedClients, images executor.Images, templates executor.Templates, serviceAccountNames map[string]string, metrics ExecutionMetric, emiter *event.Emitter, configMap config.Repository, - testsClient testsv3.Interface, clientset kubernetes.Interface, - testExecutionsClient testexecutionsv1.Interface, - templatesClient templatesv1.Interface, registry string, podStartTimeout time.Duration, clusterID string, @@ -108,7 +103,8 @@ func NewJobExecutor( return &JobExecutor{ ClientSet: clientset, - Repository: repo, + deprecatedRepositories: deprecatedRepositories, + deprecatedClients: deprecatedClients, Log: log.DefaultLogger, images: images, templates: templates, @@ -116,9 +112,6 @@ func NewJobExecutor( metrics: metrics, Emitter: emiter, configMap: configMap, - testsClient: testsClient, - testExecutionsClient: testExecutionsClient, - templatesClient: templatesClient, registry: registry, podStartTimeout: podStartTimeout, clusterID: clusterID, @@ -139,7 +132,8 @@ type ExecutionMetric interface { // JobExecutor is container for managing job executor dependencies type JobExecutor struct { - Repository result.Repository + deprecatedRepositories commons.DeprecatedRepositories + deprecatedClients commons.DeprecatedClients Log *zap.SugaredLogger ClientSet kubernetes.Interface Cmd string @@ -149,9 +143,6 @@ type JobExecutor struct { metrics ExecutionMetric Emitter *event.Emitter configMap config.Repository - testsClient testsv3.Interface - testExecutionsClient testexecutionsv1.Interface - templatesClient templatesv1.Interface registry string podStartTimeout time.Duration clusterID string @@ -339,7 +330,7 @@ func (c *JobExecutor) MonitorJobForTimeout(ctx context.Context, jobName, namespa // CreateJob creates new Kubernetes job based on execution and execute options func (c *JobExecutor) CreateJob(ctx context.Context, execution testkube.Execution, options ExecuteOptions) error { jobs := c.ClientSet.BatchV1().Jobs(execution.TestNamespace) - jobOptions, err := NewJobOptions(c.Log, c.templatesClient, c.images, c.templates, + jobOptions, err := NewJobOptions(c.Log, c.deprecatedClients.Templates(), c.images, c.templates, c.serviceAccountNames, c.registry, c.clusterID, c.apiURI, execution, options, c.natsURI, c.debug) if err != nil { return err @@ -455,7 +446,7 @@ func (c *JobExecutor) updateResultsFromPod(ctx context.Context, pod corev1.Pod, } func (c *JobExecutor) stopExecution(ctx context.Context, l *zap.SugaredLogger, execution *testkube.Execution, result *testkube.ExecutionResult, isNegativeTest bool) error { - savedExecution, err := c.Repository.Get(ctx, execution.Id) + savedExecution, err := c.deprecatedRepositories.TestResults().Get(ctx, execution.Id) if err != nil { l.Errorw("get execution error", "error", err) return err @@ -491,7 +482,7 @@ func (c *JobExecutor) stopExecution(ctx context.Context, l *zap.SugaredLogger, e result.ErrorMessage = execution.ExecutionResult.ErrorMessage } - err = c.Repository.EndExecution(ctx, *execution) + err = c.deprecatedRepositories.TestResults().EndExecution(ctx, *execution) if err != nil { l.Errorw("Update execution result error", "error", err) return err @@ -511,32 +502,32 @@ func (c *JobExecutor) stopExecution(ctx context.Context, l *zap.SugaredLogger, e // metrics increase execution.ExecutionResult = result l.Infow("execution ended, saving result", "executionId", execution.Id, "status", result.Status) - if err = c.Repository.UpdateResult(ctx, execution.Id, *execution); err != nil { + if err = c.deprecatedRepositories.TestResults().UpdateResult(ctx, execution.Id, *execution); err != nil { l.Errorw("Update execution result error", "error", err) return err } - test, err := c.testsClient.Get(execution.TestName) + test, err := c.deprecatedClients.Tests().Get(execution.TestName) if err != nil { l.Errorw("getting test error", "error", err) return err } test.Status = testsmapper.MapExecutionToTestStatus(execution) - if err = c.testsClient.UpdateStatus(test); err != nil { + if err = c.deprecatedClients.Tests().UpdateStatus(test); err != nil { l.Errorw("updating test error", "error", err) return err } if execution.TestExecutionName != "" { - testExecution, err := c.testExecutionsClient.Get(execution.TestExecutionName) + testExecution, err := c.deprecatedClients.TestExecutions().Get(execution.TestExecutionName) if err != nil { l.Errorw("getting test execution error", "error", err) return err } testExecution.Status = testexecutionsmapper.MapAPIToCRD(execution, testExecution.Generation) - if err = c.testExecutionsClient.UpdateStatus(testExecution); err != nil { + if err = c.deprecatedClients.TestExecutions().UpdateStatus(testExecution); err != nil { l.Errorw("updating test execution error", "error", err) return err } @@ -800,7 +791,7 @@ func (c *JobExecutor) Abort(ctx context.Context, execution *testkube.Execution) func (c *JobExecutor) Timeout(ctx context.Context, jobName string) (result *testkube.ExecutionResult) { l := c.Log.With("jobName", jobName) l.Infow("job timeout") - execution, err := c.Repository.Get(ctx, jobName) + execution, err := c.deprecatedRepositories.TestResults().Get(ctx, jobName) if err != nil { l.Errorw("error getting execution", "error", err) return diff --git a/pkg/executor/common.go b/pkg/executor/common.go index df812dc91e9..58a1ef7f92e 100644 --- a/pkg/executor/common.go +++ b/pkg/executor/common.go @@ -32,14 +32,8 @@ const ( // VolumeDir is volume dir VolumeDir = "/data" defaultLogLinesCount = 100 - // GitUsernameSecretName is git username secret name - GitUsernameSecretName = "git-username" - // GitTokenSecretName is git token secret name - GitTokenSecretName = "git-token" // SlavesConfigsEnv is slave configs for creating slaves in executor SlavesConfigsEnv = "RUNNER_SLAVES_CONFIGS" - - SidecarImage = "kubeshop/testkube-logs-sidecar:v0-3" // TODO - change it to valid image name after deployment will be ready ) var RunnerEnvVars = []corev1.EnvVar{ @@ -462,40 +456,12 @@ func SyncDefaultExecutors( executorsClient executorsclientv1.Interface, namespace string, executors []testkube.ExecutorDetails, - readOnlyExecutors bool, -) (images Images, err error) { +) error { if len(executors) == 0 { - return images, nil + return nil } - // TODO - remove it after merging helm templates fully - images.LogSidecar = SidecarImage - for _, executor := range executors { - - if executor.Executor == nil { - continue - } - - if executor.Name == "logs-sidecar" { - images.LogSidecar = executor.Executor.Image - continue - } - - if executor.Name == "init-executor" { - images.Init = executor.Executor.Image - continue - } - - if executor.Name == "scraper-executor" { - images.Scraper = executor.Executor.Image - continue - } - - if readOnlyExecutors { - continue - } - obj := &executorv1.Executor{ ObjectMeta: metav1.ObjectMeta{ Name: executor.Name, @@ -516,11 +482,11 @@ func SyncDefaultExecutors( result, err := executorsClient.Get(executor.Name) if err != nil && !k8serrors.IsNotFound(err) { - return images, err + return err } if err != nil { if _, err = executorsClient.Create(obj); err != nil { - return images, err + return err } } else { obj.Spec.JobTemplate = result.Spec.JobTemplate @@ -528,12 +494,12 @@ func SyncDefaultExecutors( obj.Spec.UseDataDirAsWorkingDir = result.Spec.UseDataDirAsWorkingDir result.Spec = obj.Spec if _, err = executorsClient.Update(result); err != nil { - return images, err + return err } } } - return images, nil + return nil } // GetPodErrorMessage returns pod error message diff --git a/pkg/executor/containerexecutor/containerexecutor.go b/pkg/executor/containerexecutor/containerexecutor.go index c08dfe8263f..7d8b1e5b7e5 100644 --- a/pkg/executor/containerexecutor/containerexecutor.go +++ b/pkg/executor/containerexecutor/containerexecutor.go @@ -6,6 +6,7 @@ import ( "path/filepath" "time" + "github.com/kubeshop/testkube/cmd/api-server/commons" "github.com/kubeshop/testkube/pkg/cache" "github.com/pkg/errors" @@ -16,8 +17,6 @@ import ( "github.com/kubeshop/testkube/pkg/secret" "github.com/kubeshop/testkube/pkg/utils" - "github.com/kubeshop/testkube/pkg/repository/result" - "github.com/kubeshop/testkube/pkg/version" "go.uber.org/zap" @@ -27,10 +26,6 @@ import ( "k8s.io/client-go/kubernetes" executorv1 "github.com/kubeshop/testkube-operator/api/executor/v1" - executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" - templatesv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1" - testexecutionsv1 "github.com/kubeshop/testkube-operator/pkg/client/testexecutions/v1" - testsv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/executor" "github.com/kubeshop/testkube/pkg/executor/client" @@ -64,7 +59,8 @@ type EventEmitter interface { // NewContainerExecutor creates new job executor func NewContainerExecutor( - repo result.Repository, + deprecatedRepositories commons.DeprecatedRepositories, + deprecatedClients commons.DeprecatedClients, images executor.Images, templates executor.Templates, imageInspector imageinspector.Inspector, @@ -72,10 +68,6 @@ func NewContainerExecutor( metrics ExecutionMetric, emiter EventEmitter, configMap config.Repository, - executorsClient executorsclientv1.Interface, - testsClient testsv3.Interface, - testExecutionsClient testexecutionsv1.Interface, - templatesClient templatesv1.Interface, registry string, podStartTimeout time.Duration, clusterID string, @@ -100,7 +92,8 @@ func NewContainerExecutor( return &ContainerExecutor{ clientSet: clientSet, - repository: repo, + deprecatedRepositories: deprecatedRepositories, + deprecatedClients: deprecatedClients, log: log.DefaultLogger, images: images, templates: templates, @@ -109,10 +102,6 @@ func NewContainerExecutor( serviceAccountNames: serviceAccountNames, metrics: metrics, emitter: emiter, - testsClient: testsClient, - executorsClient: executorsClient, - testExecutionsClient: testExecutionsClient, - templatesClient: templatesClient, registry: registry, podStartTimeout: podStartTimeout, clusterID: clusterID, @@ -134,7 +123,8 @@ type ExecutionMetric interface { // ContainerExecutor is container for managing job executor dependencies type ContainerExecutor struct { - repository result.Repository + deprecatedRepositories commons.DeprecatedRepositories + deprecatedClients commons.DeprecatedClients log *zap.SugaredLogger clientSet kubernetes.Interface images executor.Images @@ -144,10 +134,6 @@ type ContainerExecutor struct { emitter EventEmitter configMap config.Repository serviceAccountNames map[string]string - testsClient testsv3.Interface - executorsClient executorsclientv1.Interface - testExecutionsClient testexecutionsv1.Interface - templatesClient templatesv1.Interface registry string podStartTimeout time.Duration clusterID string @@ -221,13 +207,13 @@ func (c *ContainerExecutor) Logs(ctx context.Context, id, namespace string) (out close(out) }() - execution, err := c.repository.Get(ctx, id) + execution, err := c.deprecatedRepositories.TestResults().Get(ctx, id) if err != nil { out <- output.NewOutputError(err) return } - exec, err := c.executorsClient.GetByType(execution.TestType) + exec, err := c.deprecatedClients.Executors().GetByType(execution.TestType) if err != nil { out <- output.NewOutputError(err) return @@ -336,7 +322,7 @@ func (c *ContainerExecutor) createJob(ctx context.Context, execution testkube.Ex ) } - jobOptions, err := NewJobOptions(c.log, c.templatesClient, c.images, c.templates, inspector, + jobOptions, err := NewJobOptions(c.log, c.deprecatedClients.Templates(), c.images, c.templates, inspector, c.serviceAccountNames, c.registry, c.clusterID, c.apiURI, execution, options, c.natsURI, c.debug) if err != nil { return nil, err @@ -501,7 +487,7 @@ func (c *ContainerExecutor) updateResultsFromPod( l.Errorw("parse output error", "error", err) execution.ExecutionResult.Output = output execution.ExecutionResult.Err(err) - err = c.repository.UpdateResult(ctx, execution.Id, *execution) + err = c.deprecatedRepositories.TestResults().UpdateResult(ctx, execution.Id, *execution) if err != nil { l.Errorw("Update result error", "error", err) } @@ -529,7 +515,7 @@ func (c *ContainerExecutor) updateResultsFromPod( } l.Infow("container execution completed saving result", "executionId", execution.Id, "status", execution.ExecutionResult.Status) - err = c.repository.UpdateResult(ctx, execution.Id, *execution) + err = c.deprecatedRepositories.TestResults().UpdateResult(ctx, execution.Id, *execution) if err != nil { l.Errorw("Update execution result error", "error", err) } @@ -559,13 +545,13 @@ func (c *ContainerExecutor) stopExecution(ctx context.Context, result.Status = execution.ExecutionResult.Status result.ErrorMessage = execution.ExecutionResult.ErrorMessage - err := c.repository.UpdateResult(ctx, execution.Id, *execution) + err := c.deprecatedRepositories.TestResults().UpdateResult(ctx, execution.Id, *execution) if err != nil { c.log.Errorw("Update execution result error", "error", err) } } - err := c.repository.EndExecution(ctx, *execution) + err := c.deprecatedRepositories.TestResults().EndExecution(ctx, *execution) if err != nil { c.log.Errorw("Update execution result error", "error", err) } @@ -574,27 +560,27 @@ func (c *ContainerExecutor) stopExecution(ctx context.Context, execution.ExecutionResult = result c.metrics.IncAndObserveExecuteTest(*execution, c.dashboardURI) - test, err := c.testsClient.Get(execution.TestName) + test, err := c.deprecatedClients.Tests().Get(execution.TestName) if err != nil { c.log.Errorw("getting test error", "error", err) } if test != nil { test.Status = testsmapper.MapExecutionToTestStatus(execution) - if err = c.testsClient.UpdateStatus(test); err != nil { + if err = c.deprecatedClients.Tests().UpdateStatus(test); err != nil { c.log.Errorw("updating test error", "error", err) } } if execution.TestExecutionName != "" { - testExecution, err := c.testExecutionsClient.Get(execution.TestExecutionName) + testExecution, err := c.deprecatedClients.TestExecutions().Get(execution.TestExecutionName) if err != nil { c.log.Errorw("getting test execution error", "error", err) } if testExecution != nil { testExecution.Status = testexecutionsmapper.MapAPIToCRD(execution, testExecution.Generation) - if err = c.testExecutionsClient.UpdateStatus(testExecution); err != nil { + if err = c.deprecatedClients.TestExecutions().UpdateStatus(testExecution); err != nil { c.log.Errorw("updating test execution error", "error", err) } } diff --git a/pkg/executor/containerexecutor/containerexecutor_test.go b/pkg/executor/containerexecutor/containerexecutor_test.go index 16f5dc9786d..4b60566dfb3 100644 --- a/pkg/executor/containerexecutor/containerexecutor_test.go +++ b/pkg/executor/containerexecutor/containerexecutor_test.go @@ -17,6 +17,7 @@ import ( testsv3 "github.com/kubeshop/testkube-operator/api/tests/v3" templatesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1" v3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" + "github.com/kubeshop/testkube/cmd/api-server/commons" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/executor" "github.com/kubeshop/testkube/pkg/executor/client" @@ -30,16 +31,26 @@ var ctx = context.Background() func TestExecuteAsync(t *testing.T) { t.Parallel() + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + deprecatedRepositories := commons.NewMockDeprecatedRepositories(mockCtrl) + deprecatedRepositories.EXPECT().TestResults().Return(FakeResultRepository{}).AnyTimes() + + deprecatedClients := commons.NewMockDeprecatedClients(mockCtrl) + deprecatedClients.EXPECT().Tests().Return(FakeTestsClient{}).AnyTimes() + deprecatedClients.EXPECT().Executors().Return(FakeExecutorsClient{}).AnyTimes() + deprecatedClients.EXPECT().Templates().Return(nil).AnyTimes() + ce := ContainerExecutor{ - clientSet: getFakeClient("1"), - log: logger(), - repository: FakeResultRepository{}, - metrics: FakeExecutionMetric{}, - emitter: FakeEmitter{}, - configMap: FakeConfigRepository{}, - testsClient: FakeTestsClient{}, - executorsClient: FakeExecutorsClient{}, - serviceAccountNames: map[string]string{"": ""}, + clientSet: getFakeClient("1"), + log: logger(), + deprecatedRepositories: deprecatedRepositories, + deprecatedClients: deprecatedClients, + metrics: FakeExecutionMetric{}, + emitter: FakeEmitter{}, + configMap: FakeConfigRepository{}, + serviceAccountNames: map[string]string{"": ""}, } execution := &testkube.Execution{Id: "1"} @@ -56,16 +67,26 @@ func TestExecuteAsync(t *testing.T) { func TestExecuteSync(t *testing.T) { t.Parallel() + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + deprecatedRepositories := commons.NewMockDeprecatedRepositories(mockCtrl) + deprecatedRepositories.EXPECT().TestResults().Return(FakeResultRepository{}).AnyTimes() + + deprecatedClients := commons.NewMockDeprecatedClients(mockCtrl) + deprecatedClients.EXPECT().Tests().Return(FakeTestsClient{}).AnyTimes() + deprecatedClients.EXPECT().Executors().Return(FakeExecutorsClient{}).AnyTimes() + deprecatedClients.EXPECT().Templates().Return(nil).AnyTimes() + ce := ContainerExecutor{ - clientSet: getFakeClient("1"), - log: logger(), - repository: FakeResultRepository{}, - metrics: FakeExecutionMetric{}, - emitter: FakeEmitter{}, - configMap: FakeConfigRepository{}, - testsClient: FakeTestsClient{}, - executorsClient: FakeExecutorsClient{}, - serviceAccountNames: map[string]string{"default": ""}, + clientSet: getFakeClient("1"), + log: logger(), + deprecatedRepositories: deprecatedRepositories, + deprecatedClients: deprecatedClients, + metrics: FakeExecutionMetric{}, + emitter: FakeEmitter{}, + configMap: FakeConfigRepository{}, + serviceAccountNames: map[string]string{"default": ""}, } execution := &testkube.Execution{Id: "1", TestNamespace: "default"} diff --git a/pkg/migrator/migrator.go b/pkg/migrator/migrator.go deleted file mode 100644 index 2b16579a4b4..00000000000 --- a/pkg/migrator/migrator.go +++ /dev/null @@ -1,106 +0,0 @@ -package migrator - -import ( - "fmt" - "strings" - - "go.uber.org/zap" - - "github.com/kubeshop/testkube/pkg/log" - "github.com/kubeshop/testkube/pkg/process" - "github.com/kubeshop/testkube/pkg/semver" -) - -type Migration interface { - Migrate() error - Version() string - Info() string - Type() MigrationType -} - -// MigrationType is migration type -type MigrationType int - -const ( - // MigrationTypeClient is client migration type - MigrationTypeClient MigrationType = iota + 1 - // MigrationTypeServer is server migration type - MigrationTypeServer -) - -// NewMigrator returns new Migrator instance -func NewMigrator() *Migrator { - return &Migrator{ - Log: log.DefaultLogger, - } -} - -// Migrator struct to manage migrations of Testkube API and CRDs -type Migrator struct { - Migrations []Migration - Log *zap.SugaredLogger -} - -// Add adds new migration -func (m *Migrator) Add(migration Migration) { - m.Migrations = append(m.Migrations, migration) -} - -// GetValidMigrations returns valid migration list for currentVersion -func (m *Migrator) GetValidMigrations(currentVersion string, migrationTypes ...MigrationType) (migrations []Migration) { - types := make(map[MigrationType]struct{}, len(migrationTypes)) - for _, migrationType := range migrationTypes { - types[migrationType] = struct{}{} - } - - for _, migration := range m.Migrations { - if ok, err := m.IsValid(migration.Version(), currentVersion); ok && err == nil { - if _, ok = types[migration.Type()]; ok { - migrations = append(migrations, migration) - } - } - } - - return -} - -// Run runs migrations of passed migration types -func (m *Migrator) Run(currentVersion string, migrationTypes ...MigrationType) error { - for _, migration := range m.GetValidMigrations(currentVersion, migrationTypes...) { - err := migration.Migrate() - if err != nil { - return err - } - } - - return nil -} - -// IsValid checks if versions constraints are met, assuming that currentVersion -// is just updated version and it should be taken for migration -func (m Migrator) IsValid(migrationVersion, currentVersion string) (bool, error) { - - // clean possible v prefixes - migrationVersion = strings.TrimPrefix(migrationVersion, "v") - currentVersion = strings.TrimPrefix(currentVersion, "v") - - if migrationVersion == "" || currentVersion == "" { - return false, fmt.Errorf("empty version migration:'%s', current:'%s'", migrationVersion, currentVersion) - } - - return semver.Lte(currentVersion, migrationVersion) -} - -// ExecuteCommands executes multiple commands returns multiple commands outputs -func (m Migrator) ExecuteCommands(commands []string) (outputs []string, err error) { - for _, command := range commands { - out, err := process.ExecuteString(command) - if err != nil { - return outputs, err - } - - outputs = append(outputs, string(out)) - } - - return outputs, nil -} diff --git a/pkg/migrator/migrator_test.go b/pkg/migrator/migrator_test.go deleted file mode 100644 index 6675cc259d8..00000000000 --- a/pkg/migrator/migrator_test.go +++ /dev/null @@ -1,185 +0,0 @@ -package migrator - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -var ErrMigrationFailed = fmt.Errorf("migration failed") - -func TestMigrator(t *testing.T) { - - t.Run("migrate versions one after another", func(t *testing.T) { - // given - migrator := NewMigrator() - migrator.Add(&Migr1{}) - migrator.Add(&Migr2{}) - migrator.Add(&Migr3{}) - - // when - err := migrator.Run("0.0.2", MigrationTypeClient) - assert.NoError(t, err) - - // then - assert.Equal(t, migrator.Migrations[0].(*Migr1).Run, false) - assert.Equal(t, migrator.Migrations[1].(*Migr2).Run, true) - assert.Equal(t, migrator.Migrations[2].(*Migr3).Run, true) - }) - - t.Run("migrate mixed versions", func(t *testing.T) { - // given - migrator := NewMigrator() - migrator.Add(&Migr3{}) - migrator.Add(&Migr1{}) - migrator.Add(&Migr2{}) - migrator.Add(&Migr1{}) - - // when - err := migrator.Run("0.0.2", MigrationTypeClient) - assert.NoError(t, err) - - // then - assert.Equal(t, migrator.Migrations[0].(*Migr3).Run, true) - assert.Equal(t, migrator.Migrations[1].(*Migr1).Run, false) - assert.Equal(t, migrator.Migrations[2].(*Migr2).Run, true) - assert.Equal(t, migrator.Migrations[3].(*Migr1).Run, false) - }) - - t.Run("failed migration returns error", func(t *testing.T) { - // given - migrator := NewMigrator() - migrator.Add(&Migr1{}) - migrator.Add(&MigrFailed{}) - migrator.Add(&Migr1{}) - - // when - err := migrator.Run("0.0.1", MigrationTypeClient) - - // then - assert.Error(t, err, ErrMigrationFailed) - }) - - t.Run("run only client migration", func(t *testing.T) { - // given - migrator := NewMigrator() - migrator.Add(&Migr1{}) - migrator.Add(&MigrServer{}) - - // when - err := migrator.Run("0.0.1", MigrationTypeClient) - assert.NoError(t, err) - - // then - assert.Equal(t, migrator.Migrations[0].(*Migr1).Run, true) - assert.Equal(t, migrator.Migrations[1].(*MigrServer).Run, false) - }) - - t.Run("run only server migration", func(t *testing.T) { - // given - migrator := NewMigrator() - migrator.Add(&Migr1{}) - migrator.Add(&MigrServer{}) - - // when - err := migrator.Run("0.0.1", MigrationTypeServer) - assert.NoError(t, err) - - // then - assert.Equal(t, migrator.Migrations[0].(*Migr1).Run, false) - assert.Equal(t, migrator.Migrations[1].(*MigrServer).Run, true) - }) - -} - -type Migr1 struct { - Run bool -} - -func (m *Migr1) Version() string { - return "0.0.1" -} -func (m *Migr1) Migrate() error { - m.Run = true - return nil -} -func (m *Migr1) Info() string { - return "some migration description 1" -} - -func (m *Migr1) Type() MigrationType { - return MigrationTypeClient -} - -type Migr2 struct { - Run bool -} - -func (m *Migr2) Version() string { - return "0.0.2" -} -func (m *Migr2) Migrate() error { - m.Run = true - return nil -} -func (m *Migr2) Info() string { - return "some migration description 2" -} -func (m *Migr2) Type() MigrationType { - return MigrationTypeClient -} - -type Migr3 struct { - Run bool -} - -func (m *Migr3) Version() string { - return "0.0.3" -} -func (m *Migr3) Migrate() error { - m.Run = true - return nil -} -func (m *Migr3) Info() string { - return "some migration description 3" -} -func (m *Migr3) Type() MigrationType { - return MigrationTypeClient -} - -type MigrFailed struct { - Run bool -} - -func (m *MigrFailed) Version() string { - return "0.0.1" -} -func (m *MigrFailed) Migrate() error { - m.Run = true - return ErrMigrationFailed -} -func (m *MigrFailed) Info() string { - return "some failed migration" -} -func (m *MigrFailed) Type() MigrationType { - return MigrationTypeClient -} - -type MigrServer struct { - Run bool -} - -func (m *MigrServer) Version() string { - return "0.0.1" -} -func (m *MigrServer) Migrate() error { - m.Run = true - return nil -} -func (m *MigrServer) Info() string { - return "some server migration" -} -func (m *MigrServer) Type() MigrationType { - return MigrationTypeServer -} diff --git a/pkg/reconciler/reconciler.go b/pkg/reconciler/reconciler.go index 10b9affa093..6219a42be86 100644 --- a/pkg/reconciler/reconciler.go +++ b/pkg/reconciler/reconciler.go @@ -11,7 +11,7 @@ import ( "k8s.io/client-go/kubernetes" executorv1 "github.com/kubeshop/testkube-operator/api/executor/v1" - executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" + "github.com/kubeshop/testkube/cmd/api-server/commons" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/executor" cexecutor "github.com/kubeshop/testkube/pkg/executor/containerexecutor" @@ -30,21 +30,18 @@ var ( ) type Client struct { - k8sclient kubernetes.Interface - resultRepository result.Repository - testResultRepository testresult.Repository - executorsClient *executorsclientv1.ExecutorsClient - logger *zap.SugaredLogger + k8sclient kubernetes.Interface + deprecatedRepositories commons.DeprecatedRepositories + deprecatedClients commons.DeprecatedClients + logger *zap.SugaredLogger } -func NewClient(k8sclient kubernetes.Interface, resultRepository result.Repository, testResultRepository testresult.Repository, - executorsClient *executorsclientv1.ExecutorsClient, logger *zap.SugaredLogger) *Client { +func NewClient(k8sclient kubernetes.Interface, deprecatedRepositories commons.DeprecatedRepositories, deprecatedClients commons.DeprecatedClients, logger *zap.SugaredLogger) *Client { return &Client{ - k8sclient: k8sclient, - resultRepository: resultRepository, - testResultRepository: testResultRepository, - executorsClient: executorsClient, - logger: logger, + k8sclient: k8sclient, + deprecatedRepositories: deprecatedRepositories, + deprecatedClients: deprecatedClients, + logger: logger, } } @@ -75,7 +72,7 @@ func (client *Client) Run(ctx context.Context) error { } func (client *Client) ProcessTests(ctx context.Context) error { - executions, err := client.resultRepository.GetExecutions(ctx, + executions, err := client.deprecatedRepositories.TestResults().GetExecutions(ctx, result.NewExecutionsFilter().WithStatus(string(*testkube.ExecutionStatusRunning))) if err != nil { return err @@ -87,7 +84,7 @@ OuterLoop: case <-ctx.Done(): return ctx.Err() default: - exec, err := client.executorsClient.GetByType(execution.TestType) + exec, err := client.deprecatedClients.Executors().GetByType(execution.TestType) if err != nil { client.logger.Errorw("error getting executor by test type", "error", err) } @@ -142,7 +139,7 @@ OuterLoop: Status: testkube.ExecutionStatusFailed, ErrorMessage: errMessage, } - if err = client.resultRepository.Update(ctx, execution); err != nil { + if err = client.deprecatedRepositories.TestResults().Update(ctx, execution); err != nil { return err } } @@ -152,7 +149,7 @@ OuterLoop: } func (client *Client) ProcessTestSuites(ctx context.Context) error { - executions, err := client.testResultRepository.GetExecutions(ctx, + executions, err := client.deprecatedRepositories.TestSuiteResults().GetExecutions(ctx, testresult.NewExecutionsFilter().WithStatus(string(*testkube.TestSuiteExecutionStatusRunning))) if err != nil { return err @@ -175,7 +172,7 @@ OuterLoop: for _, execute := range step.Execute { if execute.Step != nil { if execute.Step.Type() == testkube.TestSuiteStepTypeExecuteTest && execute.Execution != nil { - exec, err := client.resultRepository.Get(ctx, execute.Execution.Id) + exec, err := client.deprecatedRepositories.TestResults().Get(ctx, execute.Execution.Id) if err != nil && err != mongo.ErrNoDocuments { return err } @@ -237,7 +234,7 @@ OuterLoop: } } } - if err = client.testResultRepository.Update(ctx, execution); err != nil { + if err = client.deprecatedRepositories.TestSuiteResults().Update(ctx, execution); err != nil { return err } } diff --git a/pkg/repository/config/configmap.go b/pkg/repository/config/configmap.go index d691c4e7bcc..451b4118d2a 100644 --- a/pkg/repository/config/configmap.go +++ b/pkg/repository/config/configmap.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strconv" + "sync" "github.com/pkg/errors" @@ -29,52 +30,86 @@ func NewConfigMapConfig(name, namespace string) (*ConfigMapConfig, error) { type ConfigMapConfig struct { name string client *configmap.Client + + data *testkube.Config + mu sync.Mutex +} + +func (c *ConfigMapConfig) getDefaultClusterId() string { + return fmt.Sprintf("cluster%s", telemetry.GetMachineID()) } // GetUniqueClusterId gets unique cluster based ID -func (c *ConfigMapConfig) GetUniqueClusterId(ctx context.Context) (clusterId string, err error) { - config, err := c.Get(ctx) - // generate new cluster Id - if config.ClusterId == "" { - return fmt.Sprintf("cluster%s", telemetry.GetMachineID()), err +func (c *ConfigMapConfig) GetUniqueClusterId(_ context.Context) (clusterId string, err error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.data == nil { + return "", errors.New("config not loaded yet") } - - return config.ClusterId, nil + return c.data.ClusterId, nil } // GetTelemetryEnabled get telemetry enabled -func (c *ConfigMapConfig) GetTelemetryEnabled(ctx context.Context) (ok bool, err error) { - config, err := c.Get(ctx) - if err != nil { - return true, err +func (c *ConfigMapConfig) GetTelemetryEnabled(_ context.Context) (bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.data == nil { + return false, errors.New("config not loaded yet") } - - return config.EnableTelemetry, nil + return c.data.EnableTelemetry, nil } // Get config -func (c *ConfigMapConfig) Get(ctx context.Context) (result testkube.Config, err error) { - data, err := c.client.Get(ctx, c.name) - if err != nil { - return result, errors.Wrap(err, "reading config map error") +func (c *ConfigMapConfig) Get(_ context.Context) (result testkube.Config, err error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.data == nil { + return result, errors.New("config not loaded yet") } + return *c.data, nil +} - result.ClusterId = data["clusterId"] - if enableTelemetry, ok := data["enableTelemetry"]; ok { - result.EnableTelemetry, err = strconv.ParseBool(enableTelemetry) - if err != nil { - return result, errors.Wrap(err, "parsing enable telemetry error") +func (c *ConfigMapConfig) load(ctx context.Context, defaultTelemetryEnabled bool) error { + // Load configuration from the ConfigMap + data, _ := c.client.Get(ctx, c.name) + c.data = &testkube.Config{} + if len(data) > 0 { + c.data.ClusterId = data["clusterId"] + if enableTelemetry, ok := data["enableTelemetry"]; ok { + c.data.EnableTelemetry, _ = strconv.ParseBool(enableTelemetry) + } else { + c.data.EnableTelemetry = defaultTelemetryEnabled } } - return + // Create new configuration if it doesn't exist + if c.data.ClusterId != "" { + c.data.ClusterId = c.getDefaultClusterId() + c.data.EnableTelemetry = defaultTelemetryEnabled + _, err := c.upsert(ctx, *c.data) + return err + } + + return nil } -// Upsert inserts record if not exists, updates otherwise -func (c *ConfigMapConfig) Upsert(ctx context.Context, result testkube.Config) (updated testkube.Config, err error) { +func (c *ConfigMapConfig) Load(ctx context.Context, defaultTelemetryEnabled bool) error { + c.mu.Lock() + defer c.mu.Unlock() + return c.load(ctx, defaultTelemetryEnabled) +} + +func (c *ConfigMapConfig) upsert(ctx context.Context, result testkube.Config) (updated testkube.Config, err error) { + c.data = &testkube.Config{ + ClusterId: result.ClusterId, + EnableTelemetry: result.EnableTelemetry, + } + if c.data.ClusterId == "" { + c.data.ClusterId = c.getDefaultClusterId() + } data := map[string]string{ - "clusterId": result.ClusterId, - "enableTelemetry": fmt.Sprint(result.EnableTelemetry), + "clusterId": c.data.ClusterId, + "enableTelemetry": fmt.Sprint(c.data.EnableTelemetry), } if err = c.client.Apply(ctx, c.name, data); err != nil { return result, errors.Wrap(err, "writing config map error") @@ -82,3 +117,10 @@ func (c *ConfigMapConfig) Upsert(ctx context.Context, result testkube.Config) (u return result, err } + +// Upsert inserts record if not exists, updates otherwise +func (c *ConfigMapConfig) Upsert(ctx context.Context, result testkube.Config) (updated testkube.Config, err error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.upsert(ctx, result) +} diff --git a/pkg/repository/config/mongo.go b/pkg/repository/config/mongo.go index 9765564fd16..1fb3e56ea32 100644 --- a/pkg/repository/config/mongo.go +++ b/pkg/repository/config/mongo.go @@ -15,6 +15,7 @@ import ( const CollectionName = "config" const Id = "api" +// TODO: Delete, as it's no longer used (may be need to kept for backwards compatibility [?]) func NewMongoRepository(db *mongo.Database, opts ...Opt) *MongoRepository { r := &MongoRepository{ Coll: db.Collection(CollectionName), diff --git a/pkg/scheduler/service.go b/pkg/scheduler/service.go index 50f032c7ad3..7e06cbf5db8 100644 --- a/pkg/scheduler/service.go +++ b/pkg/scheduler/service.go @@ -3,68 +3,52 @@ package scheduler import ( "go.uber.org/zap" + "github.com/kubeshop/testkube/cmd/api-server/commons" "github.com/kubeshop/testkube/pkg/event/bus" "github.com/kubeshop/testkube/pkg/repository/config" - executorsv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" - testsv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" - testsourcesv1 "github.com/kubeshop/testkube-operator/pkg/client/testsources/v1" - testsuiteexecutionsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testsuiteexecutions/v1" - testsuitesv3 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v3" v1 "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/pkg/configmap" "github.com/kubeshop/testkube/pkg/event" "github.com/kubeshop/testkube/pkg/executor/client" "github.com/kubeshop/testkube/pkg/featureflags" logsclient "github.com/kubeshop/testkube/pkg/logs/client" - "github.com/kubeshop/testkube/pkg/repository/result" - "github.com/kubeshop/testkube/pkg/repository/testresult" "github.com/kubeshop/testkube/pkg/secret" "github.com/kubeshop/testkube/pkg/tcl/checktcl" ) type Scheduler struct { - metrics v1.Metrics - executor client.Executor - containerExecutor client.Executor - testResults result.Repository - testsuiteResults testresult.Repository - executorsClient executorsv1.Interface - testsClient testsv3.Interface - testSuitesClient testsuitesv3.Interface - testSourcesClient testsourcesv1.Interface - secretClient secret.Interface - events *event.Emitter - logger *zap.SugaredLogger - configMap config.Repository - configMapClient configmap.Interface - testSuiteExecutionsClient testsuiteexecutionsclientv1.Interface - eventsBus bus.Bus - dashboardURI string - featureFlags featureflags.FeatureFlags - logsStream logsclient.Stream - subscriptionChecker checktcl.SubscriptionChecker - namespace string - agentAPITLSSecret string - runnerCustomCASecret string + metrics v1.Metrics + executor client.Executor + containerExecutor client.Executor + deprecatedRepositories commons.DeprecatedRepositories + deprecatedClients commons.DeprecatedClients + secretClient secret.Interface + events *event.Emitter + logger *zap.SugaredLogger + configMap config.Repository + configMapClient configmap.Interface + eventsBus bus.Bus + dashboardURI string + featureFlags featureflags.FeatureFlags + logsStream logsclient.Stream + subscriptionChecker checktcl.SubscriptionChecker + namespace string + agentAPITLSSecret string + runnerCustomCASecret string } func NewScheduler( metrics v1.Metrics, executor client.Executor, containerExecutor client.Executor, - executionResults result.Repository, - testExecutionResults testresult.Repository, - executorsClient executorsv1.Interface, - testsClient testsv3.Interface, - testSuitesClient testsuitesv3.Interface, - testSourcesClient testsourcesv1.Interface, + deprecatedRepositories commons.DeprecatedRepositories, + deprecatedClients commons.DeprecatedClients, secretClient secret.Interface, events *event.Emitter, logger *zap.SugaredLogger, configMap config.Repository, configMapClient configmap.Interface, - testSuiteExecutionsClient testsuiteexecutionsclientv1.Interface, eventsBus bus.Bus, dashboardURI string, featureFlags featureflags.FeatureFlags, @@ -72,36 +56,26 @@ func NewScheduler( namespace string, agentAPITLSSecret string, runnerCustomCASecret string, + subscriptionChecker checktcl.SubscriptionChecker, ) *Scheduler { return &Scheduler{ - metrics: metrics, - executor: executor, - containerExecutor: containerExecutor, - secretClient: secretClient, - testResults: executionResults, - testsuiteResults: testExecutionResults, - executorsClient: executorsClient, - testsClient: testsClient, - testSuitesClient: testSuitesClient, - testSourcesClient: testSourcesClient, - events: events, - logger: logger, - configMap: configMap, - configMapClient: configMapClient, - testSuiteExecutionsClient: testSuiteExecutionsClient, - eventsBus: eventsBus, - dashboardURI: dashboardURI, - featureFlags: featureFlags, - logsStream: logsStream, - namespace: namespace, - agentAPITLSSecret: agentAPITLSSecret, - runnerCustomCASecret: runnerCustomCASecret, + metrics: metrics, + executor: executor, + containerExecutor: containerExecutor, + secretClient: secretClient, + deprecatedRepositories: deprecatedRepositories, + deprecatedClients: deprecatedClients, + events: events, + logger: logger, + configMap: configMap, + configMapClient: configMapClient, + eventsBus: eventsBus, + dashboardURI: dashboardURI, + featureFlags: featureFlags, + logsStream: logsStream, + namespace: namespace, + agentAPITLSSecret: agentAPITLSSecret, + runnerCustomCASecret: runnerCustomCASecret, + subscriptionChecker: subscriptionChecker, } } - -// WithSubscriptionChecker sets subscription checker for the Scheduler -// This is used to check if Pro/Enterprise subscription is valid -func (s *Scheduler) WithSubscriptionChecker(subscriptionChecker checktcl.SubscriptionChecker) *Scheduler { - s.subscriptionChecker = subscriptionChecker - return s -} diff --git a/pkg/scheduler/test_scheduler.go b/pkg/scheduler/test_scheduler.go index 99690cb91ab..e6558719be1 100644 --- a/pkg/scheduler/test_scheduler.go +++ b/pkg/scheduler/test_scheduler.go @@ -60,14 +60,14 @@ func (s *Scheduler) executeTest(ctx context.Context, test testkube.Test, request } // test name + test execution name should be unique - execution, _ = s.testResults.GetByNameAndTest(ctx, request.Name, test.Name) + execution, _ = s.deprecatedRepositories.TestResults().GetByNameAndTest(ctx, request.Name, test.Name) if execution.Name == request.Name { err := errors.Errorf("test execution with name %s already exists", request.Name) return s.handleExecutionError(ctx, execution, "duplicate execution: %w", err) } - secretUUID, err := s.testsClient.GetCurrentSecretUUID(test.Name) + secretUUID, err := s.deprecatedClients.Tests().GetCurrentSecretUUID(test.Name) if err != nil { return s.handleExecutionError(ctx, execution, "can't get current secret uuid: %w", err) } @@ -93,7 +93,7 @@ func (s *Scheduler) executeTest(ctx context.Context, test testkube.Test, request return s.handleExecutionError(ctx, execution, "can't create secret variables `Secret` references: %w", err) } - err = s.testResults.Insert(ctx, execution) + err = s.deprecatedRepositories.TestResults().Insert(ctx, execution) if err != nil { return s.handleExecutionError(ctx, execution, "can't create new test execution, can't insert into storage: %w", err) } @@ -103,7 +103,7 @@ func (s *Scheduler) executeTest(ctx context.Context, test testkube.Test, request execution.Start() // update storage with current execution status - err = s.testResults.StartExecution(ctx, execution.Id, execution.StartTime) + err = s.deprecatedRepositories.TestResults().StartExecution(ctx, execution.Id, execution.StartTime) if err != nil { return s.handleExecutionError(ctx, execution, "can't execute test, can't insert into storage error: %w", err) } @@ -115,7 +115,7 @@ func (s *Scheduler) executeTest(ctx context.Context, test testkube.Test, request execution.ExecutionResult = result // update storage with current execution status - if uerr := s.testResults.UpdateResult(ctx, execution.Id, execution); uerr != nil { + if uerr := s.deprecatedRepositories.TestResults().UpdateResult(ctx, execution.Id, execution); uerr != nil { return s.handleExecutionError(ctx, execution, "update execution error: %w", err) } @@ -172,13 +172,13 @@ func (s *Scheduler) startTestExecution(ctx context.Context, options client.Execu } func (s *Scheduler) getExecutor(testName string) client.Executor { - testCR, err := s.testsClient.Get(testName) + testCR, err := s.deprecatedClients.Tests().Get(testName) if err != nil { s.logger.Errorw("can't get test", "test", testName, "error", err) return s.executor } - executorCR, err := s.executorsClient.GetByType(testCR.Spec.Type_) + executorCR, err := s.deprecatedClients.Executors().GetByType(testCR.Spec.Type_) if err != nil { s.logger.Errorw("can't get executor", "test", testName, "error", err) return s.executor @@ -193,7 +193,7 @@ func (s *Scheduler) getExecutor(testName string) client.Executor { } func (s *Scheduler) getNextTestExecutionNumber(testName string) int32 { - number, err := s.testResults.GetNextExecutionNumber(context.Background(), testName) + number, err := s.deprecatedRepositories.TestResults().GetNextExecutionNumber(context.Background(), testName) if err != nil { s.logger.Errorw("retrieving latest execution", "error", err) return number @@ -203,7 +203,7 @@ func (s *Scheduler) getNextTestExecutionNumber(testName string) int32 { } func (s *Scheduler) getNextTestSuiteExecutionNumber(testSuiteName string) int32 { - number, err := s.testsuiteResults.GetNextExecutionNumber(context.Background(), testSuiteName) + number, err := s.deprecatedRepositories.TestSuiteResults().GetNextExecutionNumber(context.Background(), testSuiteName) if err != nil { s.logger.Errorw("retrieving latest execution", "error", err) return number @@ -331,13 +331,13 @@ func newExecutionFromExecutionOptions(subscriptionChecker checktcl.SubscriptionC func (s *Scheduler) getExecuteOptions(namespace, id string, request testkube.ExecutionRequest) (options client.ExecuteOptions, err error) { // get test content from kubernetes CRs - testCR, err := s.testsClient.Get(id) + testCR, err := s.deprecatedClients.Tests().Get(id) if err != nil { return options, errors.Errorf("can't get test custom resource %v", err) } if testCR.Spec.Source != "" { - testSourceCR, err := s.testSourcesClient.Get(testCR.Spec.Source) + testSourceCR, err := s.deprecatedClients.TestSources().Get(testCR.Spec.Source) if err != nil { return options, errors.Errorf("cannot get test source custom resource: %v", err) } @@ -470,7 +470,7 @@ func (s *Scheduler) getExecuteOptions(namespace, id string, request testkube.Exe } // get executor from kubernetes CRs - executorCR, err := s.executorsClient.GetByType(testCR.Spec.Type_) + executorCR, err := s.deprecatedClients.Executors().GetByType(testCR.Spec.Type_) if err != nil { return options, errors.Errorf("can't get executor spec: %v", err) } diff --git a/pkg/scheduler/test_scheduler_test.go b/pkg/scheduler/test_scheduler_test.go index 5146db27275..ae95ccf1fd4 100644 --- a/pkg/scheduler/test_scheduler_test.go +++ b/pkg/scheduler/test_scheduler_test.go @@ -12,6 +12,7 @@ import ( testsv3 "github.com/kubeshop/testkube-operator/api/tests/v3" executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" testsclientv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" + "github.com/kubeshop/testkube/cmd/api-server/commons" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/configmap" "github.com/kubeshop/testkube/pkg/executor/client" @@ -70,12 +71,15 @@ func TestGetExecuteOptions(t *testing.T) { mockSecretClient := secret.NewMockInterface(mockCtrl) mockConfigMapClient := configmap.NewMockInterface(mockCtrl) + mockDeprecatedClients := commons.NewMockDeprecatedClients(mockCtrl) + mockDeprecatedClients.EXPECT().Executors().Return(mockExecutorsClient).AnyTimes() + mockDeprecatedClients.EXPECT().Tests().Return(mockTestsClient).AnyTimes() + sc := Scheduler{ - testsClient: mockTestsClient, - executorsClient: mockExecutorsClient, - logger: log.DefaultLogger, - secretClient: mockSecretClient, - configMapClient: mockConfigMapClient, + deprecatedClients: mockDeprecatedClients, + logger: log.DefaultLogger, + secretClient: mockSecretClient, + configMapClient: mockConfigMapClient, } mockTest := testsv3.Test{ diff --git a/pkg/scheduler/testsuite_scheduler.go b/pkg/scheduler/testsuite_scheduler.go index 40804b80df5..4c6bde7dc22 100644 --- a/pkg/scheduler/testsuite_scheduler.go +++ b/pkg/scheduler/testsuite_scheduler.go @@ -51,7 +51,7 @@ func (s *Scheduler) PrepareTestSuiteRequests(work []testsuitesv3.TestSuite, requ func (s *Scheduler) executeTestSuite(ctx context.Context, testSuite testkube.TestSuite, request testkube.TestSuiteExecutionRequest) ( testsuiteExecution testkube.TestSuiteExecution, err error) { s.logger.Debugw("Got testsuite to execute", "test", testSuite) - secretUUID, err := s.testSuitesClient.GetCurrentSecretUUID(testSuite.Name) + secretUUID, err := s.deprecatedClients.TestSuites().GetCurrentSecretUUID(testSuite.Name) if err != nil { return testsuiteExecution, err } @@ -119,7 +119,7 @@ func (s *Scheduler) executeTestSuite(ctx context.Context, testSuite testkube.Tes } testsuiteExecution = testkube.NewStartedTestSuiteExecution(testSuite, request) - err = s.testsuiteResults.Insert(ctx, testsuiteExecution) + err = s.deprecatedRepositories.TestSuiteResults().Insert(ctx, testsuiteExecution) if err != nil { s.logger.Errorw("Inserting test execution error", "error", err) } @@ -210,7 +210,7 @@ func (s *Scheduler) runSteps(ctx context.Context, wg *sync.WaitGroup, testsuiteE } } - err := s.testsuiteResults.Update(ctx, *testsuiteExecution) + err := s.deprecatedRepositories.TestSuiteResults().Update(ctx, *testsuiteExecution) if err != nil { s.logger.Infow("Updating test execution", "error", err) } @@ -226,7 +226,7 @@ func (s *Scheduler) runSteps(ctx context.Context, wg *sync.WaitGroup, testsuiteE s.logger.Debugw("Batch step execution result", "step", batchStepResult.Execute, "results", results) - err = s.testsuiteResults.Update(ctx, *testsuiteExecution) + err = s.deprecatedRepositories.TestSuiteResults().Update(ctx, *testsuiteExecution) if err != nil { s.logger.Errorw("saving test suite execution results error", "error", err) @@ -260,7 +260,7 @@ func (s *Scheduler) runSteps(ctx context.Context, wg *sync.WaitGroup, testsuiteE s.metrics.IncAndObserveExecuteTestSuite(*testsuiteExecution, s.dashboardURI) - err = s.testsuiteResults.Update(ctx, *testsuiteExecution) + err = s.deprecatedRepositories.TestSuiteResults().Update(ctx, *testsuiteExecution) if err != nil { s.logger.Errorw("saving final test suite execution result error", "error", err) } @@ -270,7 +270,7 @@ func (s *Scheduler) runSteps(ctx context.Context, wg *sync.WaitGroup, testsuiteE func (s *Scheduler) runAfterEachStep(ctx context.Context, execution *testkube.TestSuiteExecution, wg *sync.WaitGroup) { execution.Stop() - err := s.testsuiteResults.EndExecution(ctx, *execution) + err := s.deprecatedRepositories.TestSuiteResults().EndExecution(ctx, *execution) if err != nil { s.logger.Errorw("error setting end time", "error", err.Error()) } @@ -291,26 +291,26 @@ func (s *Scheduler) runAfterEachStep(ctx context.Context, execution *testkube.Te } if execution.TestSuite != nil { - testSuite, err := s.testSuitesClient.Get(execution.TestSuite.Name) + testSuite, err := s.deprecatedClients.TestSuites().Get(execution.TestSuite.Name) if err != nil { s.logger.Errorw("getting test suite error", "error", err) } if testSuite != nil { testSuite.Status = testsuitesmapper.MapExecutionToTestSuiteStatus(execution) - if err = s.testSuitesClient.UpdateStatus(testSuite); err != nil { + if err = s.deprecatedClients.TestSuites().UpdateStatus(testSuite); err != nil { s.logger.Errorw("updating test suite error", "error", err) } if execution.TestSuiteExecutionName != "" { - testSuiteExecution, err := s.testSuiteExecutionsClient.Get(execution.TestSuiteExecutionName) + testSuiteExecution, err := s.deprecatedClients.TestSuiteExecutions().Get(execution.TestSuiteExecutionName) if err != nil { s.logger.Errorw("getting test suite execution error", "error", err) } if testSuiteExecution != nil { testSuiteExecution.Status = testsuiteexecutionsmapper.MapAPIToCRD(execution, testSuiteExecution.Generation) - if err = s.testSuiteExecutionsClient.UpdateStatus(testSuiteExecution); err != nil { + if err = s.deprecatedClients.TestSuiteExecutions().UpdateStatus(testSuiteExecution); err != nil { s.logger.Errorw("updating test suite execution error", "error", err) } } @@ -534,7 +534,7 @@ func (s *Scheduler) executeTestStep(ctx context.Context, testsuiteExecution test } result.Start() - if err := s.testsuiteResults.Update(ctx, testsuiteExecution); err != nil { + if err := s.deprecatedRepositories.TestSuiteResults().Update(ctx, testsuiteExecution); err != nil { s.logger.Errorw("saving test suite execution start time error", "error", err) } @@ -559,7 +559,7 @@ func (s *Scheduler) executeTestStep(ctx context.Context, testsuiteExecution test if result.Execute[i].Execution.Id == r.Result.Id { result.Execute[i].Execution = &value - if err := s.testsuiteResults.Update(ctx, testsuiteExecution); err != nil { + if err := s.deprecatedRepositories.TestSuiteResults().Update(ctx, testsuiteExecution); err != nil { s.logger.Errorw("saving test suite execution results error", "error", err) } } @@ -568,7 +568,7 @@ func (s *Scheduler) executeTestStep(ctx context.Context, testsuiteExecution test } result.Stop() - if err := s.testsuiteResults.Update(ctx, testsuiteExecution); err != nil { + if err := s.deprecatedRepositories.TestSuiteResults().Update(ctx, testsuiteExecution); err != nil { s.logger.Errorw("saving test suite execution end time error", "error", err) } } diff --git a/pkg/secret/client.go b/pkg/secret/client.go index e25f562fca7..e6e3f05a891 100644 --- a/pkg/secret/client.go +++ b/pkg/secret/client.go @@ -30,7 +30,7 @@ type Interface interface { // Client provide methods to manage secrets type Client struct { - ClientSet *kubernetes.Clientset + ClientSet kubernetes.Interface Log *zap.SugaredLogger Namespace string } @@ -41,12 +41,16 @@ func NewClient(namespace string) (*Client, error) { if err != nil { return nil, err } + return NewClientFor(clientSet, namespace), nil +} +// NewClientFor is a method to create new secret client using existing clientSet +func NewClientFor(clientSet kubernetes.Interface, namespace string) *Client { return &Client{ ClientSet: clientSet, Log: log.DefaultLogger, Namespace: namespace, - }, nil + } } // Get is a method to retrieve an existing secret diff --git a/pkg/server/config.go b/pkg/server/config.go index eb0d40b8c53..39407544d7d 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -8,11 +8,8 @@ import ( // Config for HTTP server type Config struct { - Port int - Fullname string - ClusterID string - HttpBodyLimit int `envconfig:"HTTP_BODY_LIMIT"` - Http fiber.Config + Port int + Http fiber.Config } // Addr returns port based address diff --git a/pkg/server/httpserver.go b/pkg/server/httpserver.go index 46c7c2e6eda..0e25a8f033f 100644 --- a/pkg/server/httpserver.go +++ b/pkg/server/httpserver.go @@ -2,7 +2,6 @@ package server import ( "context" - "encoding/json" "net" "github.com/gofiber/adaptor/v2" @@ -12,7 +11,6 @@ import ( "go.uber.org/zap" "github.com/kubeshop/testkube/pkg/log" - "github.com/kubeshop/testkube/pkg/problem" ) // NewServer returns new HTTP server instance, initializes logger and metrics @@ -60,39 +58,29 @@ func (s *HTTPServer) Init() { } -// Warn writes RFC-7807 json problem to response -func (s *HTTPServer) Warn(c *fiber.Ctx, status int, err error, context ...interface{}) error { - c.Status(status) - c.Response().Header.Set("Content-Type", "application/problem+json") - s.Log.Warnw(err.Error(), "status", status) - pr := problem.New(status, s.getProblemMessage(err, context)) - return c.JSON(pr) -} - -// Error writes RFC-7807 json problem to response -func (s *HTTPServer) Error(c *fiber.Ctx, status int, err error, context ...interface{}) error { - c.Status(status) - c.Response().Header.Set("Content-Type", "application/problem+json") - s.Log.Errorw(err.Error(), "status", status) - pr := problem.New(status, s.getProblemMessage(err, context)) - return c.JSON(pr) -} - -// getProblemMessage creates new JSON based problem message and returns it as string -func (s *HTTPServer) getProblemMessage(err error, context ...interface{}) string { - message := err.Error() - if len(context) > 0 { - b, err := json.Marshal(context[0]) - if err == nil { - message += ", context: " + string(b) +// RoutesHandler is a handler to get existing routes +func (s *HTTPServer) RoutesHandler() fiber.Handler { + return func(c *fiber.Ctx) error { + var routes []fiber.Route + + stack := s.Mux.Stack() + for _, e := range stack { + for _, s := range e { + route := *s + routes = append(routes, route) + } } - } - return message + return c.JSON(routes) + } } // Run starts listening for incoming connetions func (s *HTTPServer) Run(ctx context.Context) error { + // Use basic routes + s.Routes.Get("/routes", s.RoutesHandler()) + + // Start server l, err := net.Listen("tcp", s.Config.Addr()) if err != nil { return err diff --git a/pkg/tcl/checktcl/subscription.go b/pkg/tcl/checktcl/subscription.go index cb426e96b80..95042e9232a 100644 --- a/pkg/tcl/checktcl/subscription.go +++ b/pkg/tcl/checktcl/subscription.go @@ -23,12 +23,15 @@ import ( ) type SubscriptionChecker struct { - proContext config.ProContext - orgPlan OrganizationPlan + orgPlan OrganizationPlan } // NewSubscriptionChecker creates a new subscription checker using the agent token func NewSubscriptionChecker(ctx context.Context, proContext config.ProContext, cloudClient cloud.TestKubeCloudAPIClient, grpcConn *grpc.ClientConn) (SubscriptionChecker, error) { + if cloudClient == nil || grpcConn == nil { + return SubscriptionChecker{}, nil + } + executor := executor.NewCloudGRPCExecutor(cloudClient, grpcConn, proContext.APIKey) req := GetOrganizationPlanRequest{} @@ -48,7 +51,7 @@ func NewSubscriptionChecker(ctx context.Context, proContext config.ProContext, c PlanStatus: PlanStatus(commandResponse.PlanStatus), } - return SubscriptionChecker{proContext: proContext, orgPlan: subscription}, nil + return SubscriptionChecker{orgPlan: subscription}, nil } // GetCurrentOrganizationPlan returns current organization plan diff --git a/pkg/triggers/executor.go b/pkg/triggers/executor.go index 9ab3cc46bb5..a1a922cae86 100644 --- a/pkg/triggers/executor.go +++ b/pkg/triggers/executor.go @@ -190,7 +190,7 @@ func (s *Service) getTests(t *testtriggersv1.TestTrigger) ([]testsv3.Test, error var tests []testsv3.Test if t.Spec.TestSelector.Name != "" { s.logger.Debugf("trigger service: executor component: fetching testsv3.Test with name %s", t.Spec.TestSelector.Name) - test, err := s.testsClient.Get(t.Spec.TestSelector.Name) + test, err := s.deprecatedClients.Tests().Get(t.Spec.TestSelector.Name) if err != nil { return nil, err } @@ -199,7 +199,7 @@ func (s *Service) getTests(t *testtriggersv1.TestTrigger) ([]testsv3.Test, error if t.Spec.TestSelector.NameRegex != "" { s.logger.Debugf("trigger service: executor component: fetching testsv3.Test with name regex %s", t.Spec.TestSelector.NameRegex) - testList, err := s.testsClient.List("") + testList, err := s.deprecatedClients.Tests().List("") if err != nil { return nil, err } @@ -223,7 +223,7 @@ func (s *Service) getTests(t *testtriggersv1.TestTrigger) ([]testsv3.Test, error } stringifiedSelector := selector.String() s.logger.Debugf("trigger service: executor component: fetching testsv3.Test with labels %s", stringifiedSelector) - testList, err := s.testsClient.List(stringifiedSelector) + testList, err := s.deprecatedClients.Tests().List(stringifiedSelector) if err != nil { return nil, err } @@ -236,7 +236,7 @@ func (s *Service) getTestSuites(t *testtriggersv1.TestTrigger) ([]testsuitesv3.T var testSuites []testsuitesv3.TestSuite if t.Spec.TestSelector.Name != "" { s.logger.Debugf("trigger service: executor component: fetching testsuitesv3.TestSuite with name %s", t.Spec.TestSelector.Name) - testSuite, err := s.testSuitesClient.Get(t.Spec.TestSelector.Name) + testSuite, err := s.deprecatedClients.TestSuites().Get(t.Spec.TestSelector.Name) if err != nil { return nil, err } @@ -245,7 +245,7 @@ func (s *Service) getTestSuites(t *testtriggersv1.TestTrigger) ([]testsuitesv3.T if t.Spec.TestSelector.NameRegex != "" { s.logger.Debugf("trigger service: executor component: fetching testsuitesv3.TestSuite with name regex %s", t.Spec.TestSelector.NameRegex) - testSuitesList, err := s.testSuitesClient.List("") + testSuitesList, err := s.deprecatedClients.TestSuites().List("") if err != nil { return nil, err } @@ -269,7 +269,7 @@ func (s *Service) getTestSuites(t *testtriggersv1.TestTrigger) ([]testsuitesv3.T } stringifiedSelector := selector.String() s.logger.Debugf("trigger service: executor component: fetching testsuitesv3.TestSuite with label %s", stringifiedSelector) - testSuitesList, err := s.testSuitesClient.List(stringifiedSelector) + testSuitesList, err := s.deprecatedClients.TestSuites().List(stringifiedSelector) if err != nil { return nil, err } diff --git a/pkg/triggers/executor_test.go b/pkg/triggers/executor_test.go index 1f62dcb61c4..ea4d2b3d84b 100644 --- a/pkg/triggers/executor_test.go +++ b/pkg/triggers/executor_test.go @@ -18,6 +18,7 @@ import ( testsuiteexecutionsv1 "github.com/kubeshop/testkube-operator/pkg/client/testsuiteexecutions/v1" testsuitesv3 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v3" testworkflowsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testworkflows/v1" + "github.com/kubeshop/testkube/cmd/api-server/commons" "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/configmap" @@ -32,6 +33,7 @@ import ( "github.com/kubeshop/testkube/pkg/repository/testresult" "github.com/kubeshop/testkube/pkg/scheduler" "github.com/kubeshop/testkube/pkg/secret" + "github.com/kubeshop/testkube/pkg/tcl/checktcl" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowexecutor" ) @@ -57,6 +59,17 @@ func TestExecute(t *testing.T) { mockConfigMapClient := configmap.NewMockInterface(mockCtrl) mockTestSuiteExecutionsClient := testsuiteexecutionsv1.NewMockInterface(mockCtrl) + mockDeprecatedClients := commons.NewMockDeprecatedClients(mockCtrl) + mockDeprecatedClients.EXPECT().Executors().Return(mockExecutorsClient).AnyTimes() + mockDeprecatedClients.EXPECT().Tests().Return(mockTestsClient).AnyTimes() + mockDeprecatedClients.EXPECT().TestSuites().Return(mockTestSuitesClient).AnyTimes() + mockDeprecatedClients.EXPECT().TestSources().Return(mockTestSourcesClient).AnyTimes() + mockDeprecatedClients.EXPECT().TestSuiteExecutions().Return(mockTestSuiteExecutionsClient).AnyTimes() + + mockDeprecatedRepositories := commons.NewMockDeprecatedRepositories(mockCtrl) + mockDeprecatedRepositories.EXPECT().TestResults().Return(mockResultRepository).AnyTimes() + mockDeprecatedRepositories.EXPECT().TestSuiteResults().Return(mockTestResultRepository).AnyTimes() + mockExecutor := client.NewMockExecutor(mockCtrl) mockEventEmitter := event.NewEmitter(bus.NewEventBusMock(), "") @@ -113,18 +126,13 @@ func TestExecute(t *testing.T) { metricsHandle, mockExecutor, mockExecutor, - mockResultRepository, - mockTestResultRepository, - mockExecutorsClient, - mockTestsClient, - mockTestSuitesClient, - mockTestSourcesClient, + mockDeprecatedRepositories, + mockDeprecatedClients, mockSecretClient, mockEventEmitter, log.DefaultLogger, configMapConfig, mockConfigMapClient, - mockTestSuiteExecutionsClient, mockBus, "", featureflags.FeatureFlags{}, @@ -132,13 +140,14 @@ func TestExecute(t *testing.T) { "", "", "", + checktcl.SubscriptionChecker{}, ) s := &Service{ - triggerStatus: make(map[statusKey]*triggerStatus), - scheduler: sched, - testsClient: mockTestsClient, - testSuitesClient: mockTestSuitesClient, - logger: log.DefaultLogger, + triggerStatus: make(map[statusKey]*triggerStatus), + scheduler: sched, + deprecatedRepositories: mockDeprecatedRepositories, + deprecatedClients: mockDeprecatedClients, + logger: log.DefaultLogger, } status := testtriggersv1.TRUE_TestTriggerConditionStatuses diff --git a/pkg/triggers/scraper.go b/pkg/triggers/scraper.go index 3ac1bcfa846..26be0cd904e 100644 --- a/pkg/triggers/scraper.go +++ b/pkg/triggers/scraper.go @@ -41,7 +41,7 @@ func (s *Service) checkForRunningTestExecutions(ctx context.Context, status *tri testExecutionIDs := status.getExecutionIDs() for _, id := range testExecutionIDs { - execution, err := s.resultRepository.Get(ctx, id) + execution, err := s.deprecatedRepositories.TestResults().Get(ctx, id) if err == mongo.ErrNoDocuments { s.logger.Warnf("trigger service: execution scraper component: no test execution found for id %s", id) status.removeExecutionID(id) @@ -61,7 +61,7 @@ func (s *Service) checkForRunningTestSuiteExecutions(ctx context.Context, status testSuiteExecutionIDs := status.getTestSuiteExecutionIDs() for _, id := range testSuiteExecutionIDs { - execution, err := s.testResultRepository.Get(ctx, id) + execution, err := s.deprecatedRepositories.TestSuiteResults().Get(ctx, id) if err == mongo.ErrNoDocuments { s.logger.Warnf("trigger service: execution scraper component: no testsuite execution found for id %s", id) status.removeTestSuiteExecutionID(id) @@ -113,7 +113,7 @@ func (s *Service) abortRunningTestExecutions(ctx context.Context, status *trigge testExecutionIDs := status.getExecutionIDs() for _, id := range testExecutionIDs { - execution, err := s.resultRepository.Get(ctx, id) + execution, err := s.deprecatedRepositories.TestResults().Get(ctx, id) if err == mongo.ErrNoDocuments { s.logger.Warnf("trigger service: execution scraper component: no test execution found for id %s", id) status.removeExecutionID(id) @@ -140,7 +140,7 @@ func (s *Service) abortRunningTestSuiteExecutions(ctx context.Context, status *t testSuiteExecutionIDs := status.getTestSuiteExecutionIDs() for _, id := range testSuiteExecutionIDs { - execution, err := s.testResultRepository.Get(ctx, id) + execution, err := s.deprecatedRepositories.TestSuiteResults().Get(ctx, id) if err == mongo.ErrNoDocuments { s.logger.Warnf("trigger service: execution scraper component: no testsuite execution found for id %s", id) status.removeTestSuiteExecutionID(id) diff --git a/pkg/triggers/scraper_test.go b/pkg/triggers/scraper_test.go index 204fb72a89b..496fe36365b 100644 --- a/pkg/triggers/scraper_test.go +++ b/pkg/triggers/scraper_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "go.mongodb.org/mongo-driver/mongo" + "github.com/kubeshop/testkube/cmd/api-server/commons" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/log" "github.com/kubeshop/testkube/pkg/repository/result" @@ -32,6 +33,10 @@ func TestService_runExecutionScraper(t *testing.T) { mockTestResultRepository := testresult.NewMockRepository(mockCtrl) mockTestWorkflowResultsRepository := testworkflow.NewMockRepository(mockCtrl) + mockDeprecatedRepositories := commons.NewMockDeprecatedRepositories(mockCtrl) + mockDeprecatedRepositories.EXPECT().TestResults().Return(mockResultRepository).AnyTimes() + mockDeprecatedRepositories.EXPECT().TestSuiteResults().Return(mockTestResultRepository).AnyTimes() + mockResultRepository.EXPECT().Get(gomock.Any(), "test-execution-1").Return(testkube.Execution{}, mongo.ErrNoDocuments) testSuiteExecutionStatus := testkube.PASSED_TestSuiteExecutionStatus mockTestSuiteExecution := testkube.TestSuiteExecution{Id: "test-suite-execution-1", Status: &testSuiteExecutionStatus} @@ -53,8 +58,7 @@ func TestService_runExecutionScraper(t *testing.T) { } s := &Service{ triggerStatus: triggerStatusMap, - resultRepository: mockResultRepository, - testResultRepository: mockTestResultRepository, + deprecatedRepositories: mockDeprecatedRepositories, testWorkflowResultsRepository: mockTestWorkflowResultsRepository, scraperInterval: 100 * time.Millisecond, logger: log.DefaultLogger, @@ -80,6 +84,10 @@ func TestService_runExecutionScraper(t *testing.T) { mockTestResultRepository := testresult.NewMockRepository(mockCtrl) mockTestWorkflowResultsRepository := testworkflow.NewMockRepository(mockCtrl) + mockDeprecatedRepositories := commons.NewMockDeprecatedRepositories(mockCtrl) + mockDeprecatedRepositories.EXPECT().TestResults().Return(mockResultRepository).AnyTimes() + mockDeprecatedRepositories.EXPECT().TestSuiteResults().Return(mockTestResultRepository).AnyTimes() + testSuiteExecutionStatus := testkube.RUNNING_TestSuiteExecutionStatus mockTestSuiteExecution := testkube.TestSuiteExecution{Id: "test-suite-execution-1", Status: &testSuiteExecutionStatus} mockTestResultRepository.EXPECT().Get(gomock.Any(), "test-suite-execution-1").Return(mockTestSuiteExecution, nil).Times(3) @@ -97,8 +105,7 @@ func TestService_runExecutionScraper(t *testing.T) { } s := &Service{ triggerStatus: triggerStatusMap, - resultRepository: mockResultRepository, - testResultRepository: mockTestResultRepository, + deprecatedRepositories: mockDeprecatedRepositories, testWorkflowResultsRepository: mockTestWorkflowResultsRepository, scraperInterval: 100 * time.Millisecond, logger: log.DefaultLogger, diff --git a/pkg/triggers/service.go b/pkg/triggers/service.go index c6182b12760..7673f76b781 100644 --- a/pkg/triggers/service.go +++ b/pkg/triggers/service.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "strings" + "sync" "time" "go.uber.org/zap" @@ -13,19 +14,15 @@ import ( testsv3 "github.com/kubeshop/testkube-operator/api/tests/v3" testsuitev3 "github.com/kubeshop/testkube-operator/api/testsuite/v3" testtriggersv1 "github.com/kubeshop/testkube-operator/api/testtriggers/v1" - executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" - testsclientv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" - testsuitesclientv3 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v3" testworkflowsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testworkflows/v1" testkubeclientsetv1 "github.com/kubeshop/testkube-operator/pkg/clientset/versioned" + "github.com/kubeshop/testkube/cmd/api-server/commons" "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/event/bus" "github.com/kubeshop/testkube/pkg/executor/client" "github.com/kubeshop/testkube/pkg/http" "github.com/kubeshop/testkube/pkg/repository/config" - "github.com/kubeshop/testkube/pkg/repository/result" - "github.com/kubeshop/testkube/pkg/repository/testresult" "github.com/kubeshop/testkube/pkg/repository/testworkflow" "github.com/kubeshop/testkube/pkg/scheduler" "github.com/kubeshop/testkube/pkg/telemetry" @@ -65,14 +62,11 @@ type Service struct { scheduler *scheduler.Scheduler clientset kubernetes.Interface testKubeClientset testkubeclientsetv1.Interface - testSuitesClient testsuitesclientv3.Interface - testsClient testsclientv3.Interface + deprecatedRepositories commons.DeprecatedRepositories + deprecatedClients commons.DeprecatedClients testWorkflowsClient testworkflowsclientv1.Interface - resultRepository result.Repository - testResultRepository testresult.Repository logger *zap.SugaredLogger configMap config.Repository - executorsClient executorsclientv1.Interface httpClient http.HttpClient testExecutor client.Executor eventsBus bus.Bus @@ -88,18 +82,15 @@ type Service struct { type Option func(*Service) func NewService( + deprecatedRepositories commons.DeprecatedRepositories, + deprecatedClients commons.DeprecatedClients, scheduler *scheduler.Scheduler, clientset kubernetes.Interface, testKubeClientset testkubeclientsetv1.Interface, - testSuitesClient testsuitesclientv3.Interface, - testsClient testsclientv3.Interface, testWorkflowsClient testworkflowsclientv1.Interface, - resultRepository result.Repository, - testResultRepository testresult.Repository, leaseBackend LeaseBackend, logger *zap.SugaredLogger, configMap config.Repository, - executorsClient executorsclientv1.Interface, testExecutor client.Executor, eventsBus bus.Bus, metrics metrics.Metrics, @@ -122,15 +113,12 @@ func NewService( scheduler: scheduler, clientset: clientset, testKubeClientset: testKubeClientset, - testSuitesClient: testSuitesClient, + deprecatedRepositories: deprecatedRepositories, + deprecatedClients: deprecatedClients, testWorkflowsClient: testWorkflowsClient, - testsClient: testsClient, - resultRepository: resultRepository, - testResultRepository: testResultRepository, leaseBackend: leaseBackend, logger: logger, configMap: configMap, - executorsClient: executorsClient, testExecutor: testExecutor, eventsBus: eventsBus, metrics: metrics, @@ -225,11 +213,21 @@ func WithDisableSecretCreation(disableSecretCreation bool) Option { func (s *Service) Run(ctx context.Context) { leaseChan := make(chan bool) - go s.runLeaseChecker(ctx, leaseChan) - - go s.runWatcher(ctx, leaseChan) - - go s.runExecutionScraper(ctx) + wg := sync.WaitGroup{} + wg.Add(3) + go func() { + s.runLeaseChecker(ctx, leaseChan) + wg.Done() + }() + go func() { + s.runWatcher(ctx, leaseChan) + wg.Done() + }() + go func() { + s.runExecutionScraper(ctx) + wg.Done() + }() + wg.Wait() } func (s *Service) addTrigger(t *testtriggersv1.TestTrigger) { @@ -296,14 +294,14 @@ func (s *Service) addTest(test *testsv3.Test) { } test.Labels[testkube.TestLabelTestType] = utils.SanitizeName(test.Spec.Type_) - executorCR, err := s.executorsClient.GetByType(test.Spec.Type_) + executorCR, err := s.deprecatedClients.Executors().GetByType(test.Spec.Type_) if err == nil { test.Labels[testkube.TestLabelExecutor] = executorCR.Name } else { s.logger.Debugw("can't get executor spec", "error", err) } - if _, err = s.testsClient.Update(test, s.disableSecretCreation); err != nil { + if _, err = s.deprecatedClients.Tests().Update(test, s.disableSecretCreation); err != nil { s.logger.Debugw("can't update test spec", "error", err) } } @@ -320,7 +318,7 @@ func (s *Service) updateTest(test *testsv3.Test) { changed = true } - executorCR, err := s.executorsClient.GetByType(test.Spec.Type_) + executorCR, err := s.deprecatedClients.Executors().GetByType(test.Spec.Type_) if err == nil { if test.Labels[testkube.TestLabelExecutor] != executorCR.Name { test.Labels[testkube.TestLabelExecutor] = executorCR.Name @@ -331,7 +329,7 @@ func (s *Service) updateTest(test *testsv3.Test) { } if changed { - if _, err = s.testsClient.Update(test, s.disableSecretCreation); err != nil { + if _, err = s.deprecatedClients.Tests().Update(test, s.disableSecretCreation); err != nil { s.logger.Debugw("can't update test spec", "error", err) } } diff --git a/pkg/triggers/service_test.go b/pkg/triggers/service_test.go index cbc079cf8ac..44ef14ab67d 100644 --- a/pkg/triggers/service_test.go +++ b/pkg/triggers/service_test.go @@ -21,6 +21,7 @@ import ( testsuitesv3 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v3" testworkflowsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testworkflows/v1" faketestkube "github.com/kubeshop/testkube-operator/pkg/clientset/versioned/fake" + "github.com/kubeshop/testkube/cmd/api-server/commons" "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/configmap" @@ -36,6 +37,7 @@ import ( "github.com/kubeshop/testkube/pkg/repository/testworkflow" "github.com/kubeshop/testkube/pkg/scheduler" "github.com/kubeshop/testkube/pkg/secret" + "github.com/kubeshop/testkube/pkg/tcl/checktcl" "github.com/kubeshop/testkube/pkg/testworkflows/executionworker/executionworkertypes" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowexecutor" ) @@ -67,6 +69,17 @@ func TestService_Run(t *testing.T) { mockTestWorkflowRepository := testworkflow.NewMockRepository(mockCtrl) mockExecutionWorkerClient := executionworkertypes.NewMockWorker(mockCtrl) + mockDeprecatedClients := commons.NewMockDeprecatedClients(mockCtrl) + mockDeprecatedClients.EXPECT().Executors().Return(mockExecutorsClient).AnyTimes() + mockDeprecatedClients.EXPECT().Tests().Return(mockTestsClient).AnyTimes() + mockDeprecatedClients.EXPECT().TestSuites().Return(mockTestSuitesClient).AnyTimes() + mockDeprecatedClients.EXPECT().TestSources().Return(mockTestSourcesClient).AnyTimes() + mockDeprecatedClients.EXPECT().TestSuiteExecutions().Return(mockTestSuiteExecutionsClient).AnyTimes() + + mockDeprecatedRepositories := commons.NewMockDeprecatedRepositories(mockCtrl) + mockDeprecatedRepositories.EXPECT().TestResults().Return(mockResultRepository).AnyTimes() + mockDeprecatedRepositories.EXPECT().TestSuiteResults().Return(mockTestResultRepository).AnyTimes() + mockExecutor := client.NewMockExecutor(mockCtrl) mockEventEmitter := event.NewEmitter(bus.NewEventBusMock(), "") @@ -131,18 +144,13 @@ func TestService_Run(t *testing.T) { testMetrics, mockExecutor, mockExecutor, - mockResultRepository, - mockTestResultRepository, - mockExecutorsClient, - mockTestsClient, - mockTestSuitesClient, - mockTestSourcesClient, + mockDeprecatedRepositories, + mockDeprecatedClients, mockSecretClient, mockEventEmitter, testLogger, configMapConfig, mockConfigMapClient, - mockTestSuiteExecutionsClient, mockBus, "", featureflags.FeatureFlags{}, @@ -150,6 +158,7 @@ func TestService_Run(t *testing.T) { "", "", "", + checktcl.SubscriptionChecker{}, ) mockLeaseBackend := NewMockLeaseBackend(mockCtrl) @@ -162,18 +171,15 @@ func TestService_Run(t *testing.T) { eventBus := bus.NewEventBusMock() metrics := metrics.NewMetrics() s := NewService( + mockDeprecatedRepositories, + mockDeprecatedClients, sched, fakeClientset, fakeTestkubeClientset, - mockTestSuitesClient, - mockTestsClient, mockTestWorkflowsClient, - mockResultRepository, - mockTestResultRepository, mockLeaseBackend, testLogger, configMapConfig, - mockExecutorsClient, mockExecutor, eventBus, metrics, @@ -186,7 +192,7 @@ func TestService_Run(t *testing.T) { WithLeaseCheckerInterval(50*time.Millisecond), ) - s.Run(ctx) + go s.Run(ctx) time.Sleep(100 * time.Millisecond) diff --git a/pkg/triggers/watcher.go b/pkg/triggers/watcher.go index 37b196257f7..b3420388759 100644 --- a/pkg/triggers/watcher.go +++ b/pkg/triggers/watcher.go @@ -310,7 +310,7 @@ func (s *Service) checkExecutionPodStatus(ctx context.Context, executionID strin return nil } - execution, err := s.resultRepository.Get(ctx, executionID) + execution, err := s.deprecatedRepositories.TestResults().Get(ctx, executionID) if err != nil { s.logger.Errorf("get execution returned an error %v while looking for execution id: %s", err, executionID) return err @@ -333,7 +333,7 @@ func (s *Service) checkExecutionPodStatus(ctx context.Context, executionID strin } execution.ExecutionResult.ErrorMessage += errorMessage - test, err := s.testsClient.Get(execution.TestName) + test, err := s.deprecatedClients.Tests().Get(execution.TestName) if err != nil { s.logger.Errorf("get test returned an error %v while looking for test name: %s", err, execution.TestName) return err @@ -345,7 +345,7 @@ func (s *Service) checkExecutionPodStatus(ctx context.Context, executionID strin execution.ExecutionResult.ErrorMessage = "" } - err = s.resultRepository.UpdateResult(ctx, executionID, execution) + err = s.deprecatedRepositories.TestResults().UpdateResult(ctx, executionID, execution) if err != nil { s.logger.Errorf("update execution result returned an error %v while storing for execution id: %s", err, executionID) return err