Add support for compose option env_file. Also, support session

environment variables.

Closes #48

ecs-cli/modules/compose/cli/ecs/app/command.go

@@ -102,7 +102,7 @@ func populate(ecsContext *ecscompose.Context, cliContext *cli.Context) {
 func createCommand(factory ProjectFactory) cli.Command {
 	return cli.Command{
 		Name:   "create",
-		Usage:  "Creates an ECS task definition from your compose file.",
+		Usage:  "Creates an ECS task definition from your compose file. Note that we do not recommend using plain text environment variables for sensitive information, such as credential data.",
 		Action: WithProject(factory, ProjectCreate, false),
 	}
 }

ecs-cli/modules/compose/cli/ecs/app/service_command.go

@@ -60,7 +60,7 @@ func serviceCommand(factory ProjectFactory) cli.Command {
 func createServiceCommand(factory ProjectFactory) cli.Command {
 	return cli.Command{
 		Name:   ecs.CreateServiceCommandName,
-		Usage:  "Creates an ECS service from your compose file. The service is created with a desired count of 0, so no containers are started by this command.",
+		Usage:  "Creates an ECS service from your compose file. The service is created with a desired count of 0, so no containers are started by this command. Note that we do not recommend using plain text environment variables for sensitive information, such as credential data.",
 		Action: WithProject(factory, ProjectCreate, true),
 		Flags:  deploymentConfigFlags(true),
 	}

ecs-cli/modules/compose/ecs/context.go

@@ -51,6 +51,9 @@ func (context *Context) open() error {
 	context.ECSClient.Initialize(context.ECSParams)
 
 	context.EC2Client = ec2client.NewEC2Client(context.ECSParams)
+
+	context.Context.EnvironmentLookup = &libcompose.OsEnvLookup{}
+	context.Context.ConfigLookup = &libcompose.FileConfigLookup{}
 	return nil
 }
 

ecs-cli/modules/compose/ecs/utils/convert_compose_yml.go

@@ -32,6 +32,7 @@ func composeOptionsInit() {
 		"dns":          true,
 		"dns_search":   true,
 		"entrypoint":   true,
+		"env_file":     true,
 		"environment":  true,
 		"extra_hosts":  true,
 		"hostname":     true,

ecs-cli/modules/compose/ecs/utils/convert_compose_yml_test.go

@@ -28,7 +28,8 @@ func TestUnmarshalComposeConfig(t *testing.T) {
 	dnsServers := []string{"1.2.3.4"}
 	dnsSearchDomains := []string{"search.example.com"}
 	entryPoint := "/code/entrypoint.sh"
-	env := []string{"RACK_ENV=development", "SESSION_SECRET=session_secret"}
+	envFiles := []string{"./common.env", "/opt/prod.env"}
+	env := []string{"RACK_ENV=development", "SESSION_PORT=session_port"}
 	extraHosts := []string{"test.local:127.10.10.10"}
 	hostname := "foobarbaz"
 	labels := map[string]string{
@@ -59,9 +60,12 @@ func TestUnmarshalComposeConfig(t *testing.T) {
    - 1.2.3.4
   dns_search: search.example.com
   entrypoint: /code/entrypoint.sh
+  env_file:
+   - ./common.env
+   - /opt/prod.env
   environment:
     RACK_ENV: development
-    SESSION_SECRET: session_secret
+    SESSION_PORT: session_port
   extra_hosts:
    - test.local:127.10.10.10
   hostname: "foobarbaz"
@@ -136,6 +140,9 @@ redis:
 	if !reflect.DeepEqual(env, webEnv) {
 		t.Errorf("Expected Environment to be [%v] but got [%v]", env, webEnv)
 	}
+	if !reflect.DeepEqual(envFiles, web.EnvFile.Slice()) {
+		t.Errorf("Expected env_file to be [%v] but got [%v]", envFiles, web.EnvFile.Slice())
+	}
 
 	if !reflect.DeepEqual(extraHosts, web.ExtraHosts) {
 		t.Errorf("Expected extraHosts to be [%v] but got [%v]", extraHosts, web.ExtraHosts)

ecs-cli/modules/compose/ecs/utils/convert_task_definition.go

@@ -19,7 +19,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/Sirupsen/logrus"
+	log "github.com/Sirupsen/logrus"
 	libcompose "github.com/aws/amazon-ecs-cli/ecs-cli/modules/compose/libcompose"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ecs"
@@ -50,7 +50,7 @@ func ConvertToTaskDefinition(context libcompose.Context,
 		containerDef := &ecs.ContainerDefinition{
 			Name: aws.String(name),
 		}
-		if err := convertToContainerDef(config, volumes, containerDef); err != nil {
+		if err := convertToContainerDef(context, config, volumes, containerDef); err != nil {
 			return nil, err
 		}
 		containerDefinitions = append(containerDefinitions, containerDef)
@@ -65,7 +65,7 @@ func ConvertToTaskDefinition(context libcompose.Context,
 
 // convertToContainerDef transforms each service in the compose yml
 // to an equivalent container definition
-func convertToContainerDef(inputCfg *libcompose.ServiceConfig,
+func convertToContainerDef(context libcompose.Context, inputCfg *libcompose.ServiceConfig,
 	volumes map[string]string, outputContDef *ecs.ContainerDefinition) error {
 	// setting memory
 	var mem int64
@@ -77,20 +77,11 @@ func convertToContainerDef(inputCfg *libcompose.ServiceConfig,
 	}
 
 	// convert environment variables
-	// TODO, read env file
-	environment := []*ecs.KeyValuePair{}
-	for _, env := range inputCfg.Environment.Slice() {
-		parts := strings.SplitN(env, "=", 2)
-		name := &parts[0]
-		var value *string
-		if len(parts) > 1 {
-			value = &parts[1]
-		}
-		environment = append(environment, &ecs.KeyValuePair{
-			Name:  name,
-			Value: value,
-		})
+	envVars, err := libcompose.GetEnvVarsFromConfig(context, inputCfg)
+	if err != nil {
+		return err
 	}
+	environment := convertToKeyValuePairs(context, envVars, *outputContDef.Name, inputCfg)
 
 	// convert port mappings
 	portMappings, err := convertToPortMappings(*outputContDef.Name, inputCfg.Ports)
@@ -167,6 +158,53 @@ func convertToContainerDef(inputCfg *libcompose.ServiceConfig,
 	return nil
 }
 
+// convertToKeyValuePairs transforms the map of environment variables into list of ecs.KeyValuePair.
+// Environment variables with only a key are resolved by reading the variable from the shell where ecs-cli is executed from.
+// TODO: use this logic to generate RunTask overrides for ecs-cli compose commands (instead of always creating a new task def)
+func convertToKeyValuePairs(context libcompose.Context, envVars []string,
+	serviceName string, inputCfg *libcompose.ServiceConfig) []*ecs.KeyValuePair {
+
+	environment := []*ecs.KeyValuePair{}
+	for _, env := range envVars {
+		parts := strings.SplitN(env, "=", 2)
+		key := parts[0]
+
+		// format: key=value
+		if len(parts) > 1 && parts[1] != "" {
+			environment = append(environment, createKeyValuePair(key, parts[1]))
+			continue
+		}
+
+		// format: key
+		// format: key=
+		if context.EnvironmentLookup != nil {
+			resolvedEnvVars := context.EnvironmentLookup.Lookup(key, serviceName, inputCfg)
+
+			// couldn't resolve env var from where the command is executed. Skip the key
+			if len(resolvedEnvVars) == 0 {
+				log.WithFields(log.Fields{"key name": key}).Warn("Skipping unresolved Environment variable...")
+				continue
+			}
+
+			// found env var values from where the command is executed
+			for _, value := range resolvedEnvVars {
+				lookupParts := strings.SplitN(value, "=", 2)
+				environment = append(environment, createKeyValuePair(key, lookupParts[1]))
+			}
+		}
+
+	}
+	return environment
+}
+
+// createKeyValuePair generates an ecs.KeyValuePair object
+func createKeyValuePair(key, value string) *ecs.KeyValuePair {
+	return &ecs.KeyValuePair{
+		Name:  aws.String(key),
+		Value: aws.String(value),
+	}
+}
+
 // convertToECSVolumes transforms the map of hostPaths to the format of ecs.Volume
 func convertToECSVolumes(hostPaths map[string]string) []*ecs.Volume {
 	output := []*ecs.Volume{}
@@ -208,7 +246,7 @@ func convertToPortMappings(serviceName string, cfgPorts []string) ([]*ecs.PortMa
 			hostPort, portErr = strconv.Atoi(parts[0])
 			containerPort, portErr = strconv.Atoi(parts[1])
 		case 3: // Format "ipAddr:hostPort:containerPort" Example "127.0.0.0.1:8000:8000"
-			logrus.WithFields(logrus.Fields{
+			log.WithFields(log.Fields{
 				"container":   serviceName,
 				"portMapping": portMapping,
 			}).Warn("Ignoring the ip address while transforming it to task definition")

ecs-cli/modules/compose/ecs/utils/convert_task_definition_test.go

@@ -15,6 +15,8 @@ package utils
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
 	"reflect"
 	"strconv"
 	"testing"
@@ -143,8 +145,8 @@ func TestConvertToTaskDefinitionWithDockerLabels(t *testing.T) {
 }
 
 func TestConvertToTaskDefinitionWithEnv(t *testing.T) {
-	envKey := "username"
-	envValue := "root"
+	envKey := "rails_env"
+	envValue := "development"
 	env := envKey + "=" + envValue
 	serviceConfig := &libcompose.ServiceConfig{
 		Environment: libcompose.NewMaporEqualSlice([]string{env}),
@@ -159,6 +161,60 @@ func TestConvertToTaskDefinitionWithEnv(t *testing.T) {
 	}
 }
 
+func TestConvertToTaskDefinitionWithEnvFromShell(t *testing.T) {
+	envKey1 := "rails_env"
+	envValue1 := "development"
+	envKey2 := "port" // skips the second one if envKey2
+	env := []string{envKey1, envKey2 + "="}
+	serviceConfig := &libcompose.ServiceConfig{
+		Environment: libcompose.NewMaporEqualSlice(env),
+	}
+
+	os.Setenv(envKey1, envValue1)
+	defer func() {
+		os.Unsetenv(envKey1)
+	}()
+
+	taskDefinition := convertToTaskDefinitionInTest(t, "name", serviceConfig)
+	containerDef := *taskDefinition.ContainerDefinitions[0]
+	if containerDef.Environment == nil || len(containerDef.Environment) != 1 {
+		t.Fatalf("Expected non empty Environment, but was [%v]", containerDef.Environment)
+	}
+	if envKey1 != aws.StringValue(containerDef.Environment[0].Name) ||
+		envValue1 != aws.StringValue(containerDef.Environment[0].Value) {
+		t.Errorf("Expected env [%s]=[%s] But was [%v]", envKey1, envValue1, containerDef.Environment)
+	}
+}
+
+func TestConvertToTaskDefinitionWithEnvFile(t *testing.T) {
+	envKey := "rails_env"
+	envValue := "development"
+	envContents := []byte(envKey + "=" + envValue)
+
+	envFile, err := ioutil.TempFile("", "example")
+	if err != nil {
+		t.Fatal("Error creating tmp file:", err)
+	}
+	defer os.Remove(envFile.Name()) // clean up
+	if _, err := envFile.Write(envContents); err != nil {
+		t.Fatal("Error writing to tmp file:", err)
+	}
+
+	serviceConfig := &libcompose.ServiceConfig{
+		EnvFile: libcompose.NewStringorslice(envFile.Name()),
+	}
+
+	taskDefinition := convertToTaskDefinitionInTest(t, "name", serviceConfig)
+	containerDef := *taskDefinition.ContainerDefinitions[0]
+	if containerDef.Environment == nil || len(containerDef.Environment) == 0 {
+		t.Fatalf("Expected non empty Environment, but was [%v]", containerDef.Environment)
+	}
+	if envKey != aws.StringValue(containerDef.Environment[0].Name) ||
+		envValue != aws.StringValue(containerDef.Environment[0].Value) {
+		t.Errorf("Expected env [%s]=[%s] But was [%v]", envKey, envValue, containerDef.Environment)
+	}
+}
+
 func TestConvertToTaskDefinitionWithPortMappings(t *testing.T) {
 	serviceConfig := &libcompose.ServiceConfig{Ports: []string{portMapping}}
 
@@ -255,7 +311,9 @@ func convertToTaskDefinitionInTest(t *testing.T, name string, serviceConfig *lib
 
 	projectName := "ProjectName"
 	context := libcompose.Context{
-		ProjectName: projectName,
+		ProjectName:       projectName,
+		EnvironmentLookup: &libcompose.OsEnvLookup{},
+		ConfigLookup:      &libcompose.FileConfigLookup{},
 	}
 	taskDefinition, err := ConvertToTaskDefinition(context, serviceConfigs)
 	if err != nil {

ecs-cli/modules/compose/libcompose/merge.go

@@ -0,0 +1,77 @@
+// This file is derived from Docker's Libcompose project, Copyright 2015 Docker, Inc.
+// The original code may be found :
+// https://github.com/docker/libcompose/blob/master/project/merge.go
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Modifications are Copyright 2015-2016 Amazon.com, Inc. or its affiliates. Licensed under the Apache License 2.0
+// - Extracted local variables
+// - Continued to use "ConfigLookup" instead of the new "ResourceLookup"
+// - Instead of introducing a new struct RawService, passed the necessary config
+
+package libcompose
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"strings"
+)
+
+// GetEnvVarsFromConfig reads the environment variables from the compose file keys : environment and env_file
+func GetEnvVarsFromConfig(context Context, inputCfg *ServiceConfig) ([]string, error) {
+
+	envVars := inputCfg.Environment.Slice()
+	envFiles := inputCfg.EnvFile.Slice()
+	if len(envFiles) == 0 {
+		return envVars, nil
+	}
+
+	composeFile := context.ComposeFile
+	if context.ConfigLookup == nil {
+		return nil, fmt.Errorf("Can not use env_file in file %s no mechanism provided to load files", composeFile)
+	}
+
+	for i := len(envFiles) - 1; i >= 0; i-- {
+		envFile := envFiles[i]
+		// Lookup envFile relative to the compose file path
+		content, _, err := context.ConfigLookup.Lookup(envFile, composeFile)
+		if err != nil {
+			return nil, err
+		}
+
+		scanner := bufio.NewScanner(bytes.NewBuffer(content))
+		for scanner.Scan() {
+			line := strings.TrimSpace(scanner.Text())
+			key := strings.SplitAfter(line, "=")[0]
+
+			found := false
+			for _, v := range envVars {
+				if strings.HasPrefix(v, key) {
+					found = true
+					break
+				}
+			}
+
+			if !found {
+				envVars = append(envVars, line)
+			}
+		}
+
+		if scanner.Err() != nil {
+			return nil, scanner.Err()
+		}
+	}
+
+	return envVars, nil
+}