fix(db): replace sequence prediction (#1764)

The original attempt was to read "last_value" of a squence to determine
the *next* Primary Key and also write it into the json blob.
    
This was error prone, and especially failed on the first startup (with
an "empty" sequence).
    
This PR changes this approach. It was necessary to add an interface
function to set the transformer ID.
    
Background: The transformer ID is required for the commit events, so
that we can easily find all commit events of one transformer.

pkg/db/db.go

@@ -73,7 +73,7 @@ type DBHandler struct {
 }
 
 type EslId int64
-type TransformerID uint
+type TransformerID EslId
 type AppStateChange string
 
 const (

services/cd-service/pkg/repository/repository.go

@@ -1024,34 +1024,29 @@ func (r *repository) ApplyTransformersInternal(ctx context.Context, transaction
 				AuthorName:  user.Name,
 				AuthorEmail: user.Email,
 			}
-
-			//The ID of the transformer needs to be inserted into the json blob of the event_sourcing_light table,
-			// SO we can't read it after writing it
-			if r.DB.ShouldUseOtherTables() {
-				value, err := r.DB.DBDiscoverCurrentEsldID(ctx, transaction)
+			if r.DB.ShouldUseEslTable() {
+				err = r.DB.DBWriteEslEventInternal(ctx, t.GetDBEventType(), transaction, t, eventMetadata)
 				if err != nil {
-					fmt.Println(err)
 					return nil, nil, nil, &TransformerBatchApplyError{
 						TransformerError: err,
 						Index:            i,
 					}
 				}
-				var id db.TransformerID
-				if value == nil {
-					id = 1
-				} else {
-					//This is now guaranteed to be the next index
-					id = db.TransformerID(uint(*value) + 1)
+				// read the last written event, so we can get the primary key (eslId):
+				internal, err := r.DB.DBReadEslEventInternal(ctx, transaction, false)
+				if err != nil {
+					return nil, nil, nil, &TransformerBatchApplyError{
+						TransformerError: err,
+						Index:            i,
+					}
 				}
-				t.SetEslID(id)
-			}
-
-			err = r.DB.DBWriteEslEventInternal(ctx, t.GetDBEventType(), transaction, t, eventMetadata)
-			if err != nil {
-				return nil, nil, nil, &TransformerBatchApplyError{
-					TransformerError: err,
-					Index:            i,
+				if internal == nil {
+					return nil, nil, nil, &TransformerBatchApplyError{
+						TransformerError: fmt.Errorf("could not find esl event that was just inserted with event type %v", t.GetDBEventType()),
+						Index:            i,
+					}
 				}
+				t.SetEslID(db.TransformerID(internal.EslId))
 			}
 
 			if msg, subChanges, err := RunTransformer(ctxWithTime, t, state, transaction); err != nil {

services/cd-service/pkg/repository/transformer.go

@@ -377,7 +377,7 @@ type CreateApplicationVersion struct {
 	DisplayVersion   string            `json:"displayVersion"`
 	WriteCommitData  bool              `json:"writeCommitData"`
 	PreviousCommit   string            `json:"previousCommit"`
-	TransformerEslID db.TransformerID  `json:"eslid"`
+	TransformerEslID db.TransformerID  `json:"-"`
 }
 
 func (c *CreateApplicationVersion) GetDBEventType() db.EventType {
@@ -1050,7 +1050,7 @@ type CreateUndeployApplicationVersion struct {
 	Authentication   `json:"-"`
 	Application      string           `json:"app"`
 	WriteCommitData  bool             `json:"writeCommitData"`
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -1208,7 +1208,7 @@ func removeCommit(fs billy.Filesystem, commitID, application string) error {
 type UndeployApplication struct {
 	Authentication   `json:"-"`
 	Application      string           `json:"app"`
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -1357,7 +1357,7 @@ type DeleteEnvFromApp struct {
 	Authentication   `json:"-"`
 	Application      string           `json:"app"`
 	Environment      string           `json:"env"`
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -1445,7 +1445,7 @@ func (u *DeleteEnvFromApp) Transform(
 
 type CleanupOldApplicationVersions struct {
 	Application      string
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 }
 
 func (c *CleanupOldApplicationVersions) GetDBEventType() db.EventType {
@@ -1568,7 +1568,7 @@ type CreateEnvironmentLock struct {
 	Environment      string           `json:"env"`
 	LockId           string           `json:"lockId"`
 	Message          string           `json:"message"`
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -1731,7 +1731,7 @@ type DeleteEnvironmentLock struct {
 	Authentication   `json:"-"`
 	Environment      string           `json:"env"`
 	LockId           string           `json:"lockId"`
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -1822,7 +1822,7 @@ type CreateEnvironmentGroupLock struct {
 	EnvironmentGroup string           `json:"env"`
 	LockId           string           `json:"lockId"`
 	Message          string           `json:"message"`
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -1869,7 +1869,7 @@ type DeleteEnvironmentGroupLock struct {
 	Authentication   `json:"-"`
 	EnvironmentGroup string           `json:"envGroup"`
 	LockId           string           `json:"lockId"`
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -1915,7 +1915,7 @@ type CreateEnvironmentApplicationLock struct {
 	Application      string           `json:"app"`
 	LockId           string           `json:"lockId"`
 	Message          string           `json:"message"`
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -2004,7 +2004,7 @@ type DeleteEnvironmentApplicationLock struct {
 	Environment      string           `json:"env"`
 	Application      string           `json:"app"`
 	LockId           string           `json:"lockId"`
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -2080,7 +2080,7 @@ type CreateEnvironmentTeamLock struct {
 	Team             string           `json:"team"`
 	LockId           string           `json:"lockId"`
 	Message          string           `json:"message"`
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -2199,7 +2199,7 @@ type DeleteEnvironmentTeamLock struct {
 	Environment      string           `json:"env"`
 	Team             string           `json:"team"`
 	LockId           string           `json:"lockId"`
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 }
 
 func (c *DeleteEnvironmentTeamLock) GetDBEventType() db.EventType {
@@ -2276,7 +2276,7 @@ type CreateEnvironment struct {
 	Authentication   `json:"-"`
 	Environment      string                   `json:"env"`
 	Config           config.EnvironmentConfig `json:"config"`
-	TransformerEslID db.TransformerID         `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID         `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -2403,7 +2403,7 @@ type DeployApplicationVersion struct {
 	WriteCommitData  bool                            `json:"writeCommitData"`
 	SourceTrain      *DeployApplicationVersionSource `json:"sourceTrain"`
 	Author           string                          `json:"author"`
-	TransformerEslID db.TransformerID                `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID                `json:"-"` // Tags the transformer with EventSourcingLight eslid
 }
 
 func (c *DeployApplicationVersion) GetDBEventType() db.EventType {
@@ -2837,7 +2837,7 @@ type ReleaseTrain struct {
 	CommitHash       string           `json:"commitHash"`
 	WriteCommitData  bool             `json:"writeCommitData"`
 	Repo             Repository       `json:"-"`
-	TransformerEslID db.TransformerID `json:"eslid"`
+	TransformerEslID db.TransformerID `json:"-"`
 }
 
 func (c *ReleaseTrain) GetDBEventType() db.EventType {
@@ -3592,7 +3592,7 @@ func (c *envReleaseTrain) Transform(
 // services" commit log.
 type skippedServices struct {
 	Messages         []string
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -3623,7 +3623,7 @@ func (c *skippedServices) Transform(
 
 type skippedService struct {
 	Message          string
-	TransformerEslID db.TransformerID `json:"eslid"` // Tags the transformer with EventSourcingLight eslid
+	TransformerEslID db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslid
 
 }
 
@@ -3634,6 +3634,7 @@ func (c *skippedService) GetDBEventType() db.EventType {
 func (c *skippedService) SetEslID(id db.TransformerID) {
 	c.TransformerEslID = id
 }
+
 func (c *skippedService) Transform(_ context.Context, _ *State, _ TransformerContext, _ *sql.Tx) (string, error) {
 	return c.Message, nil
 }

services/cd-service/pkg/repository/transformer_db_test.go

@@ -320,7 +320,7 @@ func TestTransformerWritesEslDataRoundTrip(t *testing.T) {
 			if err != nil {
 				t.Fatalf("marshal error: %v\njson: \n%s\n", err, row.EventJson)
 			}
-
+			tc.Transformer.SetEslID(0) // the eslId is not part of the json blob anymore
 			if diff := cmp.Diff(tc.Transformer, jsonInterface, protocmp.Transform()); diff != "" {
 				t.Fatalf("error mismatch (-want, +got):\n%s", diff)
 			}

services/manifest-repo-export-service/pkg/cmd/server.go

@@ -331,6 +331,7 @@ func processEslEvent(ctx context.Context, repo repository.Repository, esl *db.Es
 	if err != nil {
 		return nil, err
 	}
+	t.SetEslID(db.TransformerID(esl.EslId))
 	logger.FromContext(ctx).Sugar().Infof("read esl event of type (%s) event=%v", t.GetDBEventType(), t)
 
 	err = repo.Apply(ctx, tx, t)