HP-305 Feat/kernel shutdown time

go.sum

@@ -124,7 +124,6 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gofrs/flock v0.7.0 h1:pGFUjl501gafK9HBt1VGL1KCOd/YhIooID+xgyJCf3g=
 github.com/gofrs/flock v0.7.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
@@ -192,7 +191,6 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998 h1:ruQkWz0PK91vTVrWtzAgv3VqTMgIN1FAIvwWr5MY+GQ=
 github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210804190019-f964ff605595 h1:uNrRgpnKjTfxu4qHaZAAs3eKTYV1EzGF3dAykpnxgDE=
 github.com/google/pprof v0.0.0-20210804190019-f964ff605595/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
@@ -311,7 +309,6 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=

hatchery/ecs.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -56,9 +57,9 @@ func (input *CreateTaskDefinitionInput) Environment() []*ecs.KeyValuePair {
 // TODO: Evaluate if this is still this needed..
 func (sess *CREDS) launchEcsCluster(userName string) (*ecs.Cluster, error) {
 	svc := sess.svc
-	cluster_name := strings.ReplaceAll(Config.Config.Sidecar.Env["BASE_URL"], ".", "-") + "-cluster"
+	clusterName := strings.ReplaceAll(Config.Config.Sidecar.Env["BASE_URL"], ".", "-") + "-cluster"
 	input := &ecs.CreateClusterInput{
-		ClusterName: aws.String(cluster_name),
+		ClusterName: aws.String(clusterName),
 	}
 
 	result, err := svc.CreateCluster(input)
@@ -76,13 +77,13 @@ func (sess *CREDS) launchEcsCluster(userName string) (*ecs.Cluster, error) {
 
 func (sess *CREDS) findEcsCluster(userName string) (*ecs.Cluster, error) {
 	svc := sess.svc
-	cluster_name := strings.ReplaceAll(Config.Config.Sidecar.Env["HOSTNAME"], ".", "-") + "-cluster"
-	cluster_input := &ecs.DescribeClustersInput{
+	clusterName := strings.ReplaceAll(Config.Config.Sidecar.Env["BASE_URL"], ".", "-") + "-cluster"
+	clusterInput := &ecs.DescribeClustersInput{
 		Clusters: []*string{
-			aws.String(cluster_name),
+			aws.String(clusterName),
 		},
 	}
-	result, err := svc.DescribeClusters(cluster_input)
+	describeClusterResult, err := svc.DescribeClusters(clusterInput)
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
 			switch aerr.Code() {
@@ -95,20 +96,56 @@ func (sess *CREDS) findEcsCluster(userName string) (*ecs.Cluster, error) {
 			Config.Logger.Println(err.Error())
 		}
 	}
-	if len(result.Failures) > 0 {
-		Config.Logger.Printf("ECS cluster named %s not found", cluster_name)
-		return nil, errors.New(fmt.Sprintf("ECS cluster named %s not found", cluster_name))
+	if len(describeClusterResult.Failures) > 0 {
+		for _, failure := range describeClusterResult.Failures {
+			if *failure.Reason == "MISSING" {
+				Config.Logger.Printf("ECS cluster named %s not found, trying to create this ECS cluster", clusterName)
+				input := &ecs.CreateClusterInput{
+					ClusterName: aws.String(clusterName),
+				}
+
+				_, err := svc.CreateCluster(input)
+				if err != nil {
+					if aerr, ok := err.(awserr.Error); ok {
+						switch aerr.Code() {
+						default:
+							return nil, errors.New(fmt.Sprintf("Cannot create ECS cluster named %s: %s", clusterName, aerr.Code()))
+						}
+					}
+					return nil, errors.New(fmt.Sprintf("Cannot create ECS cluster named %s: %s", clusterName, err.Error()))
+				}
+				Config.Logger.Printf("ECS cluster %s created for user %s", clusterName, userName)
+				describeClusterResult, err = svc.DescribeClusters(clusterInput)
+				if err != nil || len(describeClusterResult.Failures) > 0 {
+					return nil, errors.New(fmt.Sprintf("Still cannot find ECS cluster named %s: %s", clusterName, err.Error()))
+				}
+				return describeClusterResult.Clusters[0], nil
+			}
+		}
+		Config.Logger.Printf("ECS cluster named %s cannot be described", clusterName)
+		return nil, errors.New(fmt.Sprintf("ECS cluster named %s cannot be described", clusterName))
 	} else {
-		return result.Clusters[0], nil
+		return describeClusterResult.Clusters[0], nil
 	}
 }
 
 // Status of workspace running in ECS
-func (sess *CREDS) statusEcsWorkspace(userName string) (*WorkspaceStatus, error) {
+func (sess *CREDS) statusEcsWorkspace(ctx context.Context, userName string, accessToken string) (*WorkspaceStatus, error) {
 	status := WorkspaceStatus{}
+	statusMap := map[string]string{
+		"ACTIVE":   "Running",
+		"DRAINING": "Terminating",
+		"STOPPED":  "Not Found",
+		"INACTIVE": "Not Found",
+	}
+	statusMessage := "INACTIVE"
+	status.Status = statusMap[statusMessage]
+	status.IdleTimeLimit = -1
+	status.LastActivityTime = -1
+
 	cluster, err := sess.findEcsCluster(userName)
 	if err != nil {
-		return nil, err
+		return &status, err
 	}
 	service, err := sess.svc.DescribeServices(&ecs.DescribeServicesInput{
 		Cluster: cluster.ClusterName,
@@ -117,19 +154,57 @@ func (sess *CREDS) statusEcsWorkspace(userName string) (*WorkspaceStatus, error)
 		},
 	})
 	if err != nil {
-		return nil, err
+		return &status, err
 	}
 
-	statusMessage := "INACTIVE"
+	var taskDefName string
 	if len(service.Services) > 0 {
 		statusMessage = *service.Services[0].Status
-	}
-
-	statusMap := map[string]string{
-		"ACTIVE":   "Running",
-		"DRAINING": "Terminating",
-		"STOPPED":  "Not Found",
-		"INACTIVE": "Not Found",
+		if statusMessage == "ACTIVE" {
+			taskDefName = *service.Services[0].TaskDefinition
+			if taskDefName == "" {
+				Config.Logger.Printf("No task definition found for user %s", userName)
+			} else {
+				desTaskDefOutput, err := sess.svc.DescribeTaskDefinition(&ecs.DescribeTaskDefinitionInput{
+					TaskDefinition: &taskDefName,
+				})
+				if err == nil {
+					containerDefs := desTaskDefOutput.TaskDefinition.ContainerDefinitions
+					if len(containerDefs) > 0 {
+						args := containerDefs[0].Command
+						if len(args) > 0 {
+							for i, arg := range args {
+								if strings.Contains(*arg, "shutdown_no_activity_timeout=") {
+									Config.Logger.Printf("Found kernel idle shutdown time in args. Attempting to get last activity time\n")
+									argSplit := strings.Split(*arg, "=")
+									idleTimeLimit, err := strconv.Atoi(argSplit[len(argSplit)-1])
+									if err == nil {
+										status.IdleTimeLimit = idleTimeLimit * 1000
+										lastActivityTime, err := getKernelIdleTimeWithContext(ctx, accessToken)
+										status.LastActivityTime = lastActivityTime
+										if err != nil {
+											Config.Logger.Println(err.Error())
+										}
+									} else {
+										Config.Logger.Println(err.Error())
+									}
+									break
+								}
+								if i == len(args)-1 {
+									Config.Logger.Printf("Unable to find kernel idle shutdown time in args\n")
+								}
+							}
+						} else {
+							Config.Logger.Printf("No env vars found for task definition %s\n", taskDefName)
+						}
+					} else {
+						Config.Logger.Printf("No container definition found for task definition %s\n", taskDefName)
+					}
+				}
+			}
+		}
+	} else {
+		Config.Logger.Printf("No service found for user %s", userName)
 	}
 
 	status.Status = statusMap[statusMessage]
@@ -188,7 +263,7 @@ func terminateEcsWorkspace(ctx context.Context, userName string, accessToken str
 						break
 					}
 					if i == len(envVars)-1 {
-						Config.Logger.Printf("Unable to fund API Key ID in env vars for user %s\n", userName)
+						Config.Logger.Printf("Unable to find API Key ID in env vars for user %s\n", userName)
 					}
 				}
 			} else {
@@ -339,11 +414,13 @@ func launchEcsWorkspace(ctx context.Context, userName string, hash string, acces
 	}
 	taskDefResult, err := svc.CreateTaskDefinition(&taskDef, userName, hash)
 	if err != nil {
+		deleteAPIKeyWithContext(ctx, accessToken, apiKey.KeyID)
 		return "", err
 	}
 
 	launchTask, err := svc.launchService(ctx, taskDefResult, userName, hash)
 	if err != nil {
+		deleteAPIKeyWithContext(ctx, accessToken, apiKey.KeyID)
 		return "", err
 	}