Support task placement via ecs-params

README.md

@@ -434,6 +434,13 @@ run_params:
       subnets: array of strings          // These should be in the same VPC and Availability Zone as your instance
       security_groups: array of strings  // These should be in the same VPC as your instance
       assign_public_ip: string           // supported values: ENABLED or DISABLED
+  task_placement:
+    strategy:
+      - type: string                     // Valid values: "spread"|"binpack"|"random"
+        field: string                    // Not valid if type is "random"
+    constraints:
+      - type: string                     // Valid values: "memberOf"|"distinctInstance"
+        expression: string               // Not valid if type is "distinctInstance"
 ```
 
 **Version**
@@ -478,6 +485,17 @@ Currently, the only parameter supported under `run_params` is `network_configura
   * `subnets`: list of subnet ids used to launch tasks. ***NOTE*** These should be in the same VPC and availability zone as the instances on which you wish to launch your tasks.
   * `security_groups`: list of securtiy-group ids used to launch tasks. ***NOTE*** These should be in the same VPC as the instances on which you wish to launch your tasks.
   * `assign_public_ip`: supported values for this field are either "ENABLED" or "DISABLED". This field is *only* used for tasks launched with Fargate launch type. If this field is present in tasks with network configuration launched with EC2 launch type, the request will fail.
+* `task_placement` is an optional field with `EC2` launch-type only (it is *not* valid for `FARGATE`). It has two subfields:
+  * `strategy`: A list of objects, with two keys. Valid keys are `type` and `field`.
+    * `type`: Valid values are `random`, `binpack`, or `spread`. If `random` is specified, the `field` key is not necessary.
+    * `field`: Valid values depend on the strategy type.
+      * For `spread`, valid values are `instanceId`, `host`, or attribute key/value pairs, e.g. `attribute:ecs.instance-type =~ t2.*`
+      * For "binpack", valid values are "cpu" or "memory".
+  * `constraint`: A list of objects, with two keys. Valid keys are `type` and `expression`.
+    * `type`: Valid values are `distinctInstance` and `memberOf`. If `distinctInstance` is specified, the `expression key is not necessary.
+    * `expression`: When `type` is `memberOf`, valid values are key/value pairs for attributes or task groups, e.g. `task:group == databases` or `attribute:color =~ green`.
+
+For more information on task placement, see [Amazon ECS TaskPlacement] (https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-placement.html).
 
 Example `ecs-params.yml` file:
 
@@ -537,6 +555,24 @@ run_params:
       assign_public_ip: ENABLED
 ```
 
+Example `ecs-params.yml` with task placement:
+
+```
+version: 1
+run_params:
+  task_placement:
+    strategy:
+      - field: memory
+        type: binpack
+      - field: attribute:ecs.availability-zone
+        type: spread
+      - type: random
+    constraints:
+      - expression: attribute:ecs.instance-type =~ t2.*
+        type: memberOf
+      - type: distinctInstance`
+```
+
 You can then start a task by calling:
 ```
 ecs-cli compose --ecs-params my-ecs-params.yml up

ecs-cli/modules/cli/compose/entity/entity_helper.go

@@ -85,6 +85,8 @@ func SetupTaskDefinitionCache() cache.Cache {
 // GetOrCreateTaskDefinition gets the task definition from cache if present, else
 // creates it in ECS and persists in a local cache. It also sets the latest
 // taskDefinition to the current instance of task
+// TODO: convert to method on entity, since it changes state of entity?
+// Also, since this is called before other task/service API calls, might be good to add Fargate validation here
 func GetOrCreateTaskDefinition(entity ProjectEntity) (*ecs.TaskDefinition, error) {
 	taskDefinition := entity.TaskDefinition()
 	log.WithFields(log.Fields{

ecs-cli/modules/cli/compose/entity/service/service.go

@@ -364,12 +364,22 @@ func (s *Service) EntityType() types.Type {
 func (s *Service) buildCreateServiceInput(serviceName, taskDefName string) (*ecs.CreateServiceInput, error) {
 	launchType := s.Context().CommandConfig.LaunchType
 	cluster := s.Context().CommandConfig.Cluster
+	ecsParams := s.ecsContext.ECSParams
 
-	networkConfig, err := composeutils.ConvertToECSNetworkConfiguration(s.ecsContext.ECSParams)
+	networkConfig, err := composeutils.ConvertToECSNetworkConfiguration(ecsParams)
+	if err != nil {
+		return nil, err
+	}
+	placementConstraints, err := composeutils.ConvertToECSPlacementConstraints(ecsParams)
+	if err != nil {
+		return nil, err
+	}
+	placementStrategy, err := composeutils.ConvertToECSPlacementStrategy(ecsParams)
 	if err != nil {
 		return nil, err
 	}
 
+	// NOTE: this validation is not useful if called after GetOrCreateTaskDefinition()
 	if err = entity.ValidateFargateParams(s.Context().ECSParams, launchType); err != nil {
 		return nil, err
 	}
@@ -396,6 +406,14 @@ func (s *Service) buildCreateServiceInput(serviceName, taskDefName string) (*ecs
 		createServiceInput.NetworkConfiguration = networkConfig
 	}
 
+	if placementConstraints != nil {
+		createServiceInput.PlacementConstraints = placementConstraints
+	}
+
+	if placementStrategy != nil {
+		createServiceInput.PlacementStrategy = placementStrategy
+	}
+
 	if launchType != "" {
 		createServiceInput.LaunchType = aws.String(launchType)
 	}

ecs-cli/modules/cli/compose/entity/service/service_test.go

@@ -205,6 +205,69 @@ func TestCreateEC2Explicitly(t *testing.T) {
 	)
 }
 
+func TestCreateWithTaskPlacement(t *testing.T) {
+	flagSet := flag.NewFlagSet("ecs-cli-up", 0)
+
+	createServiceTest(
+		t,
+		flagSet,
+		&config.CommandConfig{},
+		ecsParamsWithTaskPlacement(),
+		func(input *ecs.CreateServiceInput) {
+			placementConstraints := input.PlacementConstraints
+			placementStrategy := input.PlacementStrategy
+			expectedConstraints := []*ecs.PlacementConstraint{
+				{
+					Type: aws.String("distinctInstance"),
+				}, {
+					Expression: aws.String("attribute:ecs.instance-type =~ t2.*"),
+					Type:       aws.String("memberOf"),
+				},
+			}
+			expectedStrategy := []*ecs.PlacementStrategy{
+				{
+					Type: aws.String("random"),
+				}, {
+					Field: aws.String("instanceId"),
+					Type:  aws.String("binpack"),
+				},
+			}
+
+			assert.Len(t, placementConstraints, 2)
+			assert.Equal(t, expectedConstraints, placementConstraints, "Expected Placement Constraints to match")
+			assert.Len(t, placementStrategy, 2)
+			assert.Equal(t, expectedStrategy, placementStrategy, "Expected Placement Strategy to match")
+		},
+	)
+}
+
+func ecsParamsWithTaskPlacement() *utils.ECSParams {
+	return &utils.ECSParams{
+		RunParams: utils.RunParams{
+			TaskPlacement: utils.TaskPlacement{
+				Constraints: []utils.Constraint{
+					utils.Constraint{
+						Type: ecs.PlacementConstraintTypeDistinctInstance,
+					},
+					utils.Constraint{
+						Expression: "attribute:ecs.instance-type =~ t2.*",
+						Type:       ecs.PlacementConstraintTypeMemberOf,
+					},
+				},
+				Strategies: []utils.Strategy{
+					utils.Strategy{
+						Type: ecs.PlacementStrategyTypeRandom,
+					},
+					utils.Strategy{
+						Field: "instanceId",
+						Type:  ecs.PlacementStrategyTypeBinpack,
+					},
+				},
+			},
+		},
+	}
+}
+
 // Specifies TargeGroupArn to test ALB
 func TestCreateWithALB(t *testing.T) {
 	targetGroupArn := "targetGroupArn"

ecs-cli/modules/cli/compose/entity/task/task.go

@@ -17,6 +17,7 @@ import (
 	"github.com/aws/amazon-ecs-cli/ecs-cli/modules/cli/compose/context"
 	"github.com/aws/amazon-ecs-cli/ecs-cli/modules/cli/compose/entity"
 	"github.com/aws/amazon-ecs-cli/ecs-cli/modules/cli/compose/entity/types"
+	"github.com/aws/amazon-ecs-cli/ecs-cli/modules/commands/flags"
 	"github.com/aws/amazon-ecs-cli/ecs-cli/modules/utils"
 	"github.com/aws/amazon-ecs-cli/ecs-cli/modules/utils/cache"
 	composeutils "github.com/aws/amazon-ecs-cli/ecs-cli/modules/utils/compose"
@@ -99,7 +100,8 @@ func (t *Task) Start() error {
 // if count of running tasks = 0, starts 1
 // if count != 0, and the task definitions differed, then its stops the old ones and starts the new ones
 func (t *Task) Up() error {
-	return t.up(true)
+	updateTasks := t.Context().CLIContext.Bool(flags.ForceUpdateFlag)
+	return t.up(updateTasks)
 }
 
 // Info returns a formatted list of containers (running and stopped) in the current cluster
@@ -108,19 +110,20 @@ func (t *Task) Info(filterLocal bool) (project.InfoSet, error) {
 	return entity.Info(t, filterLocal)
 }
 
-// Scale finds out the current count of running tasks for this project and scales to the desired count
+// Scale finds out the current count of running tasks for this project and scales to the desired count.
+// Any run params specified will be taken into account.
 // if desired = current, noop
 // if desired > current, stops the extra ones
 // if desired < current, start new ones (also if current was 0, create a new task definition)
-func (t *Task) Scale(expectedCount int) error {
+func (t *Task) Scale(desiredCount int) error {
 	ecsTasks, err := entity.CollectTasksWithStatus(t, ecs.DesiredStatusRunning, true)
 	if err != nil {
 		return err
 	}
 
 	observedCount := len(ecsTasks)
 
-	if expectedCount == observedCount {
+	if desiredCount == observedCount {
 		// NoOp
 		log.WithFields(log.Fields{
 			"countOfRunningTasks": observedCount,
@@ -129,18 +132,18 @@ func (t *Task) Scale(expectedCount int) error {
 		return nil
 	}
 
-	// running more than expected, stop the tasks
-	if expectedCount < observedCount {
-		diff := observedCount - expectedCount
+	// running more than desired, stop the extra tasks
+	if desiredCount < observedCount {
+		diff := observedCount - desiredCount
 		ecsTasksToStop := []*ecs.Task{}
 		for i := 0; i < diff; i++ {
 			ecsTasksToStop = append(ecsTasksToStop, ecsTasks[i])
 		}
 		return t.stopTasks(ecsTasksToStop)
 	}
 
-	// if expected > observed, then run the difference
-	diff := expectedCount - observedCount
+	// if desired > observed, then run the difference
+	diff := desiredCount - observedCount
 
 	var taskDef string
 	// if nothing was running, create new task definition
@@ -250,6 +253,7 @@ func (t *Task) stopTasks(ecsTasks []*ecs.Task) error {
 }
 
 // runTasks issues run task request to ECS Service in chunks of count=10
+// it always takes into account the latest ECS params
 func (t *Task) runTasks(taskDefinition string, totalCount int) ([]*ecs.Task, error) {
 	result := []*ecs.Task{}
 	chunkSize := 10 // can issue only up to 10 tasks in a RunTask Call
@@ -302,6 +306,7 @@ func convertToECSTaskOverride(overrides map[string][]string) (*ecs.TaskOverride,
 	return ecsOverrides, nil
 }
 
+// buildRunTaskInput will account for what is currently specified in ECS Params
 func (t *Task) buildRunTaskInput(taskDefinition string, count int, overrides map[string][]string) (*ecs.RunTaskInput, error) {
 	cluster := t.Context().CommandConfig.Cluster
 	launchType := t.Context().CommandConfig.LaunchType
@@ -314,6 +319,17 @@ func (t *Task) buildRunTaskInput(taskDefinition string, count int, overrides map
 		return nil, err
 	}
 
+	placementConstraints, err := composeutils.ConvertToECSPlacementConstraints(ecsParams)
+	if err != nil {
+		return nil, err
+	}
+
+	placementStrategy, err := composeutils.ConvertToECSPlacementStrategy(ecsParams)
+	if err != nil {
+		return nil, err
+	}
+
+	// NOTE: this validation is not useful if called after RegisterTaskDefinition
 	if err := entity.ValidateFargateParams(ecsParams, launchType); err != nil {
 		return nil, err
 	}
@@ -338,6 +354,14 @@ func (t *Task) buildRunTaskInput(taskDefinition string, count int, overrides map
 		runTaskInput.Overrides = taskOverride
 	}
 
+	if placementConstraints != nil {
+		runTaskInput.PlacementConstraints = placementConstraints
+	}
+
+	if placementStrategy != nil {
+		runTaskInput.PlacementStrategy = placementStrategy
+	}
+
 	if launchType != "" {
 		runTaskInput.LaunchType = aws.String(launchType)
 	}
@@ -354,10 +378,11 @@ func (t *Task) createOne() error {
 	return t.waitForRunTasks(ecsTask)
 }
 
-// up gets a list of running tasks and if updateTasks is set to true, it updates it with the latest task definition
-// if count of running tasks = 0, starts 1
-// if count != 0, and the task definitions differed, then its stops the old ones and starts the new ones
-func (t *Task) up(updateTasks bool) error {
+// up gets a list of running tasks. If there are no running tasks, it starts 1 task.
+// If there are no running tasks, and either the task definition has changed or
+// forceUpdate is specified, then the running tasks are stopped and relaunched
+// with the task definition and run parameters in the current call.
+func (t *Task) up(forceUpdate bool) error {
 	ecsTasks, err := entity.CollectTasksWithStatus(t, ecs.DesiredStatusRunning, true)
 	if err != nil {
 		return err
@@ -388,7 +413,7 @@ func (t *Task) up(updateTasks bool) error {
 
 	ecsTaskArns := make(map[string]bool)
 
-	if oldTaskDef != newTaskDef {
+	if oldTaskDef != newTaskDef || forceUpdate {
 		log.WithFields(log.Fields{"taskDefinition": newTaskDef}).Info("Updating to new task definition")
 
 		chunkSize := 10

ecs-cli/modules/clients/aws/ecs/client.go

@@ -224,8 +224,9 @@ func (c *ecsClient) RegisterTaskDefinition(request *ecs.RegisterTaskDefinitionIn
 func (c *ecsClient) RegisterTaskDefinitionIfNeeded(
 	request *ecs.RegisterTaskDefinitionInput,
 	taskDefinitionCache cache.Cache) (*ecs.TaskDefinition, error) {
+
 	if request.Family == nil {
-		return nil, errors.New("invalid task definitions: family is required")
+		return nil, errors.New("invalid task definition: family is required")
 	}
 
 	taskDefResp, err := c.DescribeTaskDefinition(aws.StringValue(request.Family))

ecs-cli/modules/clients/aws/ecs/client_test.go

@@ -607,6 +607,54 @@ func TestRunTask_WithTaskNetworking(t *testing.T) {
 	assert.NoError(t, err, "Unexpected error when calling RunTask")
 }
 
+func TestRunTask_WithTaskPlacement(t *testing.T) {
+	mockEcs, _, client, ctrl := setupTestController(t, getDefaultCLIConfigParams(t))
+	defer ctrl.Finish()
+
+	td := "taskDef"
+	group := "taskGroup"
+	count := 5
+
+	placementConstraints := []*ecs.PlacementConstraint{
+		{
+			Type: aws.String("distinctInstance"),
+		}, {
+			Expression: aws.String("attribute:ecs.instance-type =~ t2.*"),
+			Type:       aws.String("memberOf"),
+		},
+	}
+	placementStrategy := []*ecs.PlacementStrategy{
+		{
+			Type: aws.String("random"),
+		}, {
+			Field: aws.String("instanceId"),
+			Type:  aws.String("binpack"),
+		},
+	}
+
+	mockEcs.EXPECT().RunTask(gomock.Any()).Do(func(input interface{}) {
+		req := input.(*ecs.RunTaskInput)
+		assert.Equal(t, clusterName, aws.StringValue(req.Cluster), "Expected clusterName to match")
+		assert.Equal(t, td, aws.StringValue(req.TaskDefinition), "Expected taskDefinition to match")
+		assert.Equal(t, group, aws.StringValue(req.Group), "Expected group to match")
+		assert.Equal(t, int64(count), aws.Int64Value(req.Count), "Expected count to match")
+		assert.Equal(t, placementConstraints, req.PlacementConstraints, "Expected placement constraints to match")
+		assert.Equal(t, placementStrategy, req.PlacementStrategy, "Expected placement strategy to match")
+	}).Return(&ecs.RunTaskOutput{}, nil)
+
+	runTaskInput := &ecs.RunTaskInput{
+		Cluster:              aws.String(clusterName),
+		TaskDefinition:       aws.String(td),
+		Group:                aws.String(group),
+		Count:                aws.Int64(int64(count)),
+		LaunchType:           aws.String("EC2"),
+		PlacementConstraints: placementConstraints,
+		PlacementStrategy:    placementStrategy,
+	}
+	_, err := client.RunTask(runTaskInput)
+	assert.NoError(t, err, "Unexpected error when calling RunTask")
+}
+
 func TestIsActiveCluster(t *testing.T) {
 	mockEcs, _, client, ctrl := setupTestController(t, getDefaultCLIConfigParams(t))
 	defer ctrl.Finish()

ecs-cli/modules/commands/compose/compose_command.go

@@ -132,7 +132,7 @@ func upCommand(factory composeFactory.ProjectFactory) cli.Command {
 		Name:         "up",
 		Usage:        "Creates an ECS task definition from your compose file (if it does not already exist) and runs one instance of that task on your cluster (a combination of create and start).",
 		Action:       compose.WithProject(factory, compose.ProjectUp, false),
-		Flags:        append(flags.OptionalConfigFlags(), flags.OptionalLaunchTypeFlag(), flags.OptionalCreateLogsFlag()),
+		Flags:        append(flags.OptionalConfigFlags(), flags.OptionalLaunchTypeFlag(), flags.OptionalCreateLogsFlag(), flags.OptionalForceUpdateFlag()),
 		OnUsageError: flags.UsageErrorFactory("up"),
 	}
 }