Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: [TKC-3016] add defaultConfigs for TestWorkflow executions #6090

Merged
merged 1 commit into from
Dec 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions api/v1/testkube.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9534,6 +9534,9 @@ components:
defaultValue:
type: string
description: configuration value default
truncated:
type: boolean
description: indicates if the value is truncated


TestWorkflowConfigValue:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,6 @@ type TestWorkflowExecutionConfigValue struct {
Value string `json:"value,omitempty"`
// configuration value default
DefaultValue string `json:"defaultValue,omitempty"`
// indicates if the value is truncated
Truncated bool `json:"truncated,omitempty"`
}
54 changes: 48 additions & 6 deletions pkg/repository/testworkflow/mongo.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@ import (

var _ Repository = (*MongoRepository)(nil)

const CollectionName = "testworkflowresults"
const (
CollectionName = "testworkflowresults"
configParamSizeLimit = 100
)

func NewMongoRepository(db *mongo.Database, allowDiskUse bool, opts ...MongoRepositoryOpt) *MongoRepository {
r := &MongoRepository{
Expand Down Expand Up @@ -60,9 +63,39 @@ type MongoRepositoryOpt func(*MongoRepository)

func (r *MongoRepository) Get(ctx context.Context, id string) (result testkube.TestWorkflowExecution, err error) {
err = r.Coll.FindOne(ctx, bson.M{"$or": bson.A{bson.M{"id": id}, bson.M{"name": id}}}).Decode(&result)

if result.ResolvedWorkflow != nil && result.ResolvedWorkflow.Spec != nil && result.ConfigParams != nil {
r.populateConfigParams(result.ResolvedWorkflow, result.ConfigParams)
}

return *result.UnscapeDots(), err
}

func (r *MongoRepository) populateConfigParams(resolvedWorkflow *testkube.TestWorkflow, configParams map[string]testkube.TestWorkflowExecutionConfigValue) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not important: I don't think it should be part of this struct, as it's completely independent

for k, v := range resolvedWorkflow.Spec.Config {
if v.Default_ != nil {
if _, ok := configParams[k]; !ok {
configParams[k] = testkube.TestWorkflowExecutionConfigValue{
DefaultValue: v.Default_.Value,
}
} else {
value := configParams[k].Value
truncated := false
if len(value) > configParamSizeLimit {
value = value[:configParamSizeLimit]
truncated = true
}
configParams[k] = testkube.TestWorkflowExecutionConfigValue{
DefaultValue: v.Default_.Value,
Value: value,
Truncated: truncated,
}
}
}
}
return
}

func (r *MongoRepository) GetByNameAndTestWorkflow(ctx context.Context, name, workflowName string) (result testkube.TestWorkflowExecution, err error) {
err = r.Coll.FindOne(ctx, bson.M{"$or": bson.A{bson.M{"id": name}, bson.M{"name": name}}, "workflow.name": workflowName}).Decode(&result)
return *result.UnscapeDots(), err
Expand Down Expand Up @@ -241,8 +274,13 @@ func (r *MongoRepository) GetExecutions(ctx context.Context, filter Filter) (res
return
}

type TestWorkflowExecutionSummaryWithResolvedWorkflow struct {
testkube.TestWorkflowExecutionSummary `json:",inline" bson:",inline"`
ResolvedWorkflow *testkube.TestWorkflow `json:"resolvedWorkflow,omitempty"`
}

func (r *MongoRepository) GetExecutionsSummary(ctx context.Context, filter Filter) (result []testkube.TestWorkflowExecutionSummary, err error) {
result = make([]testkube.TestWorkflowExecutionSummary, 0)
executions := make([]TestWorkflowExecutionSummaryWithResolvedWorkflow, 0)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Better to avoid fetching unnecessary data there. You can either - change the query to fetch only important part, or even better - adjust the aggregate to provide these data.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, we have different pipelines for Enterprise, so it's very important to test it through.

query, opts := composeQueryAndOpts(filter)
if r.allowDiskUse {
opts.SetAllowDiskUse(r.allowDiskUse)
Expand All @@ -255,16 +293,20 @@ func (r *MongoRepository) GetExecutionsSummary(ctx context.Context, filter Filte
"result.steps": 0,
"result.initialization": 0,
"workflow.spec": 0,
"resolvedWorkflow": 0,
})
cursor, err := r.Coll.Find(ctx, query, opts)
if err != nil {
return
}
err = cursor.All(ctx, &result)
err = cursor.All(ctx, &executions)
result = make([]testkube.TestWorkflowExecutionSummary, len(executions))
for i := range executions {
executions[i].UnscapeDots()

for i := range result {
result[i].UnscapeDots()
if executions[i].ResolvedWorkflow != nil && executions[i].ResolvedWorkflow.Spec != nil && executions[i].ConfigParams != nil {
r.populateConfigParams(executions[i].ResolvedWorkflow, executions[i].ConfigParams)
}
result[i] = executions[i].TestWorkflowExecutionSummary
}
return
}
Expand Down
132 changes: 132 additions & 0 deletions pkg/repository/testworkflow/mongo_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -431,3 +431,135 @@ func TestNewMongoRepository_GetExecutions_Actor_Integration(t *testing.T) {

assert.Len(t, res, 0)
}

func TestNewMongoRepository_GetExecutionsSummary_Integration(t *testing.T) {
test.IntegrationTest(t)
ctx := context.Background()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(cfg.APIMongoDSN))
if err != nil {
t.Fatalf("error connecting to mongo: %v", err)
}
db := client.Database("testworkflow-executions-summary-mongo-repository-test")
t.Cleanup(func() {
db.Drop(ctx)
})
repo := NewMongoRepository(db, false)

// Insert test data
execution := testkube.TestWorkflowExecution{
Id: "test-id-1",
Name: "test-name-1",
Workflow: &testkube.TestWorkflow{
Name: "test-workflow-1",
Spec: &testkube.TestWorkflowSpec{},
},
ResolvedWorkflow: &testkube.TestWorkflow{
Name: "test-workflow-1",
Spec: &testkube.TestWorkflowSpec{
Config: map[string]testkube.TestWorkflowParameterSchema{
"param1": {
Default_: &testkube.BoxedString{
Value: "default",
},
},
},
},
},
ConfigParams: map[string]testkube.TestWorkflowExecutionConfigValue{},
}
err = repo.Insert(ctx, execution)
assert.NoError(t, err)

execution2 := testkube.TestWorkflowExecution{
Id: "test-id-1",
Name: "test-name-2",
Workflow: &testkube.TestWorkflow{
Name: "test-workflow-2",
Spec: &testkube.TestWorkflowSpec{},
},
ResolvedWorkflow: &testkube.TestWorkflow{
Name: "test-workflow-2",
Spec: &testkube.TestWorkflowSpec{
Config: map[string]testkube.TestWorkflowParameterSchema{
"param1": {
Default_: &testkube.BoxedString{
Value: "default",
},
},
},
},
},
ConfigParams: map[string]testkube.TestWorkflowExecutionConfigValue{
"param1": {
Value: "custom-value",
},
},
}
err = repo.Insert(ctx, execution2)
assert.NoError(t, err)

// Test GetExecutionsSummary
filter := NewExecutionsFilter().WithName("test-workflow-1")
result, err := repo.GetExecutionsSummary(ctx, filter)
assert.NoError(t, err)
assert.Len(t, result, 1)
assert.Equal(t, "test-name-1", result[0].Name)
assert.Equal(t, "default", result[0].ConfigParams["param1"].DefaultValue)

filter = NewExecutionsFilter().WithName("test-workflow-2")
result, err = repo.GetExecutionsSummary(ctx, filter)
assert.NoError(t, err)
assert.Len(t, result, 1)
assert.Equal(t, "test-name-2", result[0].Name)
assert.Equal(t, "default", result[0].ConfigParams["param1"].DefaultValue)
assert.Equal(t, "custom-value", result[0].ConfigParams["param1"].Value)
}

func TestNewMongoRepository_Get_Integration(t *testing.T) {
test.IntegrationTest(t)

ctx := context.Background()

client, err := mongo.Connect(ctx, options.Client().ApplyURI(cfg.APIMongoDSN))
if err != nil {
t.Fatalf("error connecting to mongo: %v", err)
}
db := client.Database("testworkflow-get-mongo-repository-test")
t.Cleanup(func() {
db.Drop(ctx)
})

repo := NewMongoRepository(db, false)
execution := testkube.TestWorkflowExecution{
Id: "test-id-1",
Name: "test-name-1",
Workflow: &testkube.TestWorkflow{
Name: "test-workflow-1",
Spec: &testkube.TestWorkflowSpec{},
},
ResolvedWorkflow: &testkube.TestWorkflow{
Name: "test-workflow-1",
Spec: &testkube.TestWorkflowSpec{
Config: map[string]testkube.TestWorkflowParameterSchema{
"param1": {
Default_: &testkube.BoxedString{
Value: "default",
},
},
},
},
},
ConfigParams: map[string]testkube.TestWorkflowExecutionConfigValue{},
}
err = repo.Insert(ctx, execution)
assert.NoError(t, err)

result, err := repo.Get(ctx, "test-id-1")
assert.NoError(t, err)

assert.Equal(t, execution.Id, result.Id)
assert.Equal(t, execution.Name, result.Name)
assert.Equal(t, "default", result.ConfigParams["param1"].DefaultValue)
assert.Equal(t, false, result.ConfigParams["param1"].Truncated)

}
17 changes: 8 additions & 9 deletions pkg/testworkflows/testworkflowexecutor/executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -468,31 +468,30 @@ func (e *executor) initialize(ctx context.Context, workflow *testworkflowsv1.Tes
DisableWebhooks: request.DisableWebhooks,
Tags: map[string]string{},
RunningContext: request.RunningContext,
ConfigParams: make(map[string]testkube.TestWorkflowExecutionConfigValue),
}

// Store the configuration if it is small and not sensitive
if testworkflows.CountMapBytes(request.Config) < ConfigSizeLimit {
storeConfig := true
schema := workflow.Spec.Config
config := make(map[string]testkube.TestWorkflowExecutionConfigValue)
for _, v := range schema {
if v.Sensitive {
storeConfig = false
execution.ConfigParams = nil
break
}
}

for k, v := range request.Config {
if _, ok := schema[k]; ok {
config[k] = testkube.TestWorkflowExecutionConfigValue{
Value: v,
if storeConfig {
for k, v := range request.Config {
if _, ok := schema[k]; ok {
execution.ConfigParams[k] = testkube.TestWorkflowExecutionConfigValue{
Value: v,
}
}
}
}

if storeConfig {
execution.ConfigParams = config
}
}

// Try to resolve tags initialily
Expand Down
Loading