Merge pull request #328 from sgotti/replace_datamanager_with_sql

*: replace etcd and datamanager with a sql db

.agola/config.jsonnet

@@ -36,7 +36,7 @@ local task_build_go(version, arch) = {
     { type: 'run', command: 'golangci-lint run --deadline 5m' },
     { type: 'run', name: 'build docker/k8s drivers tests binary', command: 'CGO_ENABLED=0 go test -c ./internal/services/executor/driver -o ./bin/docker-tests' },
     { type: 'run', name: 'build integration tests binary', command: 'go test -tags "sqlite_unlock_notify" -c ./tests -o ./bin/integration-tests' },
-    { type: 'run', name: 'run tests', command: 'SKIP_DOCKER_TESTS=1 SKIP_K8S_TESTS=1 go test -v -count 1 $(go list ./... | grep -v /tests)' },
+    { type: 'run', name: 'run tests', command: 'SKIP_DOCKER_TESTS=1 SKIP_K8S_TESTS=1 go test -tags "sqlite_unlock_notify" -v -count 1 $(go list ./... | grep -v /tests)' },
     { type: 'run', name: 'fetch gitea binary for integration tests', command: 'curl -L https://github.com/go-gitea/gitea/releases/download/v1.15.11/gitea-1.15.11-linux-amd64 -o ./bin/gitea && chmod +x ./bin/gitea' },
     { type: 'save_to_workspace', contents: [{ source_dir: './bin', dest_dir: '/bin/', paths: ['*'] }] },
   ],

Makefile

@@ -58,6 +58,14 @@ gocovmerge:
 webbundle/bindata.go: go-bindata $(WEBDISTPATH)
 	./tools/bin/go-bindata -o webbundle/bindata.go -tags webbundle -pkg webbundle -prefix "$(WEBDISTPATH)" -nocompress=true "$(WEBDISTPATH)/..."
 
+.PHONY: generate
+generate: generators
+	go generate ./...
+
+.PHONY: generators
+generators:
+	GOBIN=$(PROJDIR)/tools/bin go install ./internal/generators
+
 .PHONY: docker-agola
 docker-agola:
 	docker build --target agola . -t agola

cmd/agola/cmd/migrate.go

@@ -0,0 +1,96 @@
+// Copyright 2022 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"os"
+
+	"agola.io/agola/internal/errors"
+	"agola.io/agola/internal/migration"
+
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+var cmdMigrate = &cobra.Command{
+	Use:   "migrate",
+	Short: "migrate from an old data format export to the new data format",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := migrate(cmd, args); err != nil {
+			log.Fatal().Err(err).Send()
+		}
+	},
+}
+
+type migrateOptions struct {
+	serviceName string
+	inFilePath  string
+	outFilePath string
+}
+
+var migrateOpts migrateOptions
+
+func init() {
+	flags := cmdMigrate.Flags()
+
+	flags.StringVar(&migrateOpts.serviceName, "service", "", "service name (runservice or configstore)")
+	flags.StringVar(&migrateOpts.inFilePath, "in", "-", "input file path")
+	flags.StringVar(&migrateOpts.outFilePath, "out", "-", "output file path")
+
+	cmdAgola.AddCommand(cmdMigrate)
+}
+
+func migrate(cmd *cobra.Command, args []string) error {
+	if migrateOpts.serviceName != "runservice" && migrateOpts.serviceName != "configstore" {
+		return errors.Errorf("service option must be runservice or configstore")
+	}
+
+	var r *os.File
+	if migrateOpts.inFilePath == "-" {
+		r = os.Stdin
+	} else {
+		var err error
+		r, err = os.Open(migrateOpts.inFilePath)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+	}
+
+	var w *os.File
+	if migrateOpts.outFilePath == "-" {
+		w = os.Stdout
+	} else {
+		var err error
+		w, err = os.Create(migrateOpts.outFilePath)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+	}
+
+	log.Info().Msgf("migrating %s", migrateOpts.serviceName)
+	switch migrateOpts.serviceName {
+	case "runservice":
+		if err := migration.MigrateRunService(context.Background(), r, w); err != nil {
+			return errors.WithStack(err)
+		}
+	case "configstore":
+		if err := migration.MigrateConfigStore(context.Background(), r, w); err != nil {
+			return errors.WithStack(err)
+		}
+	}
+
+	return nil
+}

cmd/agola/cmd/serve.go

@@ -33,7 +33,6 @@ import (
 
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
-	"go.etcd.io/etcd/embed"
 )
 
 var (
@@ -64,10 +63,8 @@ var cmdServe = &cobra.Command{
 }
 
 type serveOptions struct {
-	config              string
-	components          []string
-	embeddedEtcd        bool
-	embeddedEtcdDataDir string
+	config     string
+	components []string
 }
 
 var serveOpts serveOptions
@@ -77,8 +74,6 @@ func init() {
 
 	flags.StringVar(&serveOpts.config, "config", "./config.yml", "config file path")
 	flags.StringSliceVar(&serveOpts.components, "components", []string{}, `list of components to start. Specify "all-base" to start all base components (excluding the executor).`)
-	flags.BoolVar(&serveOpts.embeddedEtcd, "embedded-etcd", false, "start and use an embedded etcd, only for testing purpose")
-	flags.StringVar(&serveOpts.embeddedEtcdDataDir, "embedded-etcd-data-dir", "/tmp/agola/etcd", "embedded etcd data dir, only for testing purpose")
 
 	if err := cmdServe.MarkFlagRequired("components"); err != nil {
 		log.Fatal().Err(err).Send()
@@ -87,30 +82,6 @@ func init() {
 	cmdAgola.AddCommand(cmdServe)
 }
 
-func embeddedEtcd(ctx context.Context) error {
-	cfg := embed.NewConfig()
-	cfg.Dir = serveOpts.embeddedEtcdDataDir
-	cfg.Logger = "zap"
-	cfg.LogOutputs = []string{"stderr"}
-
-	log.Info().Msgf("starting embedded etcd server")
-	e, err := embed.StartEtcd(cfg)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	go func() {
-		<-e.Server.ReadyNotify()
-		log.Info().Msgf("embedded etcd server is ready")
-
-		<-ctx.Done()
-		log.Info().Msgf("stopping embedded etcd server")
-		e.Close()
-	}()
-
-	return nil
-}
-
 func isComponentEnabled(name string) bool {
 	if util.StringInSlice(serveOpts.components, "all-base") && name != "executor" {
 		return true
@@ -135,12 +106,6 @@ func serve(cmd *cobra.Command, args []string) error {
 		return errors.Wrapf(err, "config error")
 	}
 
-	if serveOpts.embeddedEtcd {
-		if err := embeddedEtcd(ctx); err != nil {
-			return errors.Wrapf(err, "failed to start run service scheduler")
-		}
-	}
-
 	var rs *rsscheduler.Runservice
 	if isComponentEnabled("runservice") {
 		rs, err = rsscheduler.NewRunservice(ctx, log.Logger, &c.Runservice)

doc/devel.md

@@ -27,13 +27,7 @@ make
 
 ### Start the agola server
 
-- Copy the `example/config.yml` where you prefer
-
-```
-./bin/agola serve --embedded-etcd --config /path/to/your/config.yml --components all-base,executor
-```
-
-or use an external etcd (set it in the config.yml):
+- Copy the `example/agolademo/config.yml` where you prefer
 
 ```
 ./bin/agola serve --config /path/to/your/config.yml --components all-base,executor

doc/migrating_from_v0.7.x.md

@@ -0,0 +1,44 @@
+## Migrating from v0.7.x
+
+Agola versions after v0.7.x moved their internal db based on etcd and objectstorage to a standard external sql database (PostgreSQL or sqlite for single node deployments).
+
+If you are going to update from a version <= v0.7.x you should do some manual steps to migrate the runservice and configstore data.
+
+We suggest to test this migration on a test environment before doing this on your primary environment and keep backups.
+
+1. Ensure you're using or update to the latest v0.7.x
+1. Place somewhere the new agola > v0.7.x binary. In the next steps it'll be places in `/tmp`
+1. Keep only the runservice and configstore services active. Stop the agola gateway to avoid external activity that will be lost by the backups taken in the next steps.
+1. Take runservice and configstore backups
+
+   `curl -v http://$RUNSERVICEHOST:PORT/api/v1alpha/export > /tmp/runservice-export`
+
+   `curl -v http://$CONFIGSTOREHOST:PORT/api/v1alpha/export > /tmp/configstore-export`
+
+1. Generate the migrated data using the new agola binary migrate command:
+
+   `cat /tmp/runservice-export | ./tmp/agola migrate --service runservice > /tmp/runservice-migrated`
+
+   `cat /tmp/runservice-export | ./tmp/agola migrate --service configstore > /tmp/configstore-migrated`
+
+1. Update the agola binaries on your environment or use a test enviroment and start only the runservice and configstore.
+1. Update the agola config file and remove the runservice, configstore, notification service etcd entries and add the db entries. Every component should have its own dedicated database. DO NOT use the same database for all the services. For PostgresSQL it can be the same postgres instance but with different databases.
+1. Put the runservice and configstore in maintenance mode
+
+   `curl -v -XPUT http://$NEWRUNSERVICEHOST:PORT/api/v1alpha/maintenance`
+
+   `curl -v -XPUT http://$NEWCONFIGSTOREHOST:PORT/api/v1alpha/maintenance`
+
+1. Import the migrated data
+
+   `cat /tmp/runservice-migrated | curl -v -d @- http://$NEWRUNSERVICEHOST:PORT/api/v1alpha/import`
+
+   `cat /tmp/configstore-migrated | curl -v -d @- http://$NEWCONFIGSTOREHOST:PORT/api/v1alpha/import`
+
+1. Put the runservice and configstore in maintenance mode
+
+   `curl -v -XDELETE http://$NEWRUNSERVICEHOST:PORT/api/v1alpha/maintenance`
+
+   `curl -v -XDELETE http://$NEWCONFIGSTOREHOST:PORT/api/v1alpha/maintenance`
+
+1. Start the gateway and test if the migration was successfull

examples/agolademo/config.yml

@@ -24,13 +24,15 @@ notification:
   webExposedURL: "http://172.17.0.1:8000"
   runserviceURL: "http://localhost:4000"
   configstoreURL: "http://localhost:4002"
-  etcd:
-    endpoints: "http://localhost:2379"
+  db:
+    type: sqlite3
+    connString: /data/agola/notification/db
 
 configstore:
   dataDir: /data/agola/configstore
-  etcd:
-    endpoints: "http://localhost:2379"
+  db:
+    type: sqlite3
+    connString: /data/agola/configstore/db
   objectStorage:
     type: posix
     path: /data/agola/configstore/ost
@@ -40,8 +42,9 @@ configstore:
 runservice:
   #debug: true
   dataDir: /data/agola/runservice
-  etcd:
-    endpoints: "http://localhost:2379"
+  db:
+    type: sqlite3
+    connString: /data/agola/runservice/db
   objectStorage:
     type: posix
     path: /data/agola/runservice/ost

examples/kubernetes/distributed/README.md

@@ -4,12 +4,10 @@ This is a distributed deployment where all the components are replicated to achi
 
 Users should use it as an example base setup and change/improve it based on their needs (choosing which object storage to use).
 
-* point to an external etcd cluster
-* points to an external s3 object storage.
-* create 4 deployments for the various components with multiple replicas:
-  * runservice
-  * executor
-  * configstore
-  * gateway / scheduler
-
-
+- point to external postgres databases (one for configstore, one for runservice and one for notification service)
+- points to an external s3 object storage.
+- create 4 deployments for the various components with multiple replicas:
+  - runservice
+  - executor
+  - configstore
+  - gateway / scheduler