Merge pull request #1 from heyealex/community

Rebase from upstream/develop

README.md

@@ -92,6 +92,7 @@ appear similar to:
 
 ```shell
 terraform -chdir=hpc-cluster-small/primary init
+terraform -chdir=hpc-cluster-small/primary validate
 terraform -chdir=hpc-cluster-small/primary apply
 ```
 
@@ -136,6 +137,8 @@ you can use the following command to deploy a Packer-based resource group:
 
 ```shell
 cd <blueprint-directory>/<packer-group>/<custom-vm-image>
+packer init .
+packer validate .
 packer build .
 ```
 

cmd/create.go

@@ -18,6 +18,7 @@ limitations under the License.
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	"hpc-toolkit/pkg/config"
 	"hpc-toolkit/pkg/reswriter"
@@ -33,18 +34,25 @@ func init() {
 		"Configuration file for the new blueprints")
 	cobra.CheckErr(createCmd.Flags().MarkDeprecated("config",
 		"please see the command usage for more details."))
+
 	createCmd.Flags().StringVarP(&bpDirectory, "out", "o", "",
 		"Output directory for the new blueprints")
 	createCmd.Flags().StringSliceVar(&cliVariables, "vars", nil, msgCLIVars)
 	createCmd.Flags().StringVarP(&validationLevel, "validation-level", "l", "WARNING",
 		validationLevelDesc)
+	createCmd.Flags().BoolVarP(&overwriteBlueprint, "overwrite-blueprint", "w", false,
+		"if set, an existing blueprint dir can be overwritten by the created blueprint. \n"+
+			"Note: Terraform state IS preserved. \n"+
+			"Note: Terraform workspaces are NOT supported (behavior undefined). \n"+
+			"Note: Packer is NOT supported.")
 	rootCmd.AddCommand(createCmd)
 }
 
 var (
 	yamlFilename        string
 	bpDirectory         string
 	cliVariables        []string
+	overwriteBlueprint  bool
 	validationLevel     string
 	validationLevelDesc = "Set validation level to one of (\"ERROR\", \"WARNING\", \"IGNORE\")"
 	createCmd           = &cobra.Command{
@@ -73,5 +81,12 @@ func runCreateCmd(cmd *cobra.Command, args []string) {
 		log.Fatal(err)
 	}
 	blueprintConfig.ExpandConfig()
-	reswriter.WriteBlueprint(&blueprintConfig.Config, bpDirectory)
+	if err := reswriter.WriteBlueprint(&blueprintConfig.Config, bpDirectory, overwriteBlueprint); err != nil {
+		var target *reswriter.OverwriteDeniedError
+		if errors.As(err, &target) {
+			fmt.Printf("\n%s\n", err.Error())
+		} else {
+			log.Fatal(err)
+		}
+	}
 }

cmd/root.go

@@ -34,7 +34,7 @@ HPC deployments on the Google Cloud Platform.`,
 				log.Fatalf("cmd.Help function failed: %s", err)
 			}
 		},
-		Version: "v0.5.0-alpha (private preview)",
+		Version: "v0.6.0-alpha (private preview)",
 	}
 )
 

examples/README.md

@@ -5,7 +5,7 @@ to create a blueprint.
 
 ## Instructions
 
-Ensure your project_id is set and other deployment variables such as zone and
+Ensure your project\_id is set and other deployment variables such as zone and
 region are set correctly under `vars` before creating and deploying an example
 config.
 
@@ -154,10 +154,11 @@ All nodes mount a filestore instance on `/home`.
 
 ### image-builder.yaml
 
-This Blueprint helps create custom VM images by applying necessary software and
-configurations to existing images, such as the [HPC VM Image][hpcimage].
-Using a custom VM image can be more scalable than installing software using
-boot-time startup scripts because
+This Blueprint uses the [Packer template resource][pkr] to create custom VM
+images by applying software and configurations to existing images. By default,
+it uses the [HPC VM Image][hpcimage] as a source image. Using a custom VM image
+can be more scalable than installing software using boot-time startup scripts
+because
 
 * it avoids reliance on continued availability of package repositories
 * VMs will join an HPC cluster and execute workloads more rapidly due to reduced
@@ -167,11 +168,12 @@ boot-time startup scripts because
   relative to other based upon their creation time!
 
 [hpcimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm
+[pkr]: ../resources/packer/custom-image/README.md
 
-**Note**: it is important _not to modify_ the subnetwork name in either of the
-two resource groups without modifying them both. These _must_ match!
+**Note**:  this example relies on the default behavior of the Toolkit to derive
+naming convention for networks and other resources from the `deployment_name`.
 
-#### Custom Network (resource group)
+#### Custom Network (resource group 1)
 
 A tool called [Packer](https://packer.io) builds custom VM images by creating
 short-lived VMs, executing scripts on them, and saving the boot disk as an
@@ -189,12 +191,36 @@ connections without exposing the machine to the internet on a public IP address.
 [cloudnat]: https://cloud.google.com/nat/docs/overview
 [iap]: https://cloud.google.com/iap/docs/using-tcp-forwarding
 
-#### Packer Template (resource group)
+#### Toolkit Runners (resource group 1)
 
-The Packer template in this resource group accepts a list of Ansible playbooks
-which will be run on the VM to customize it.  Although it defaults to creating
-VMs with a public IP address, it can be easily set to use [IAP][iap] for SSH
-tunneling following the [example in its README](../resources/packer/custom-image/README.md).
+The Toolkit [startup-script](../resources/scripts/startup-script/README.md)
+module supports boot-time configuration of VMs using "runners." Runners are
+configured as a series of scripts uploaded to Cloud Storage. A simple, standard
+[VM startup script][cloudstartup] runs at boot-time, downloads the scripts from
+Cloud Storage and executes them in sequence.
+
+The standard bash startup script is exported as a string by the startup-script
+module.
+
+[vmstartup]: https://cloud.google.com/compute/docs/instances/startup-scripts/linux
+
+#### Packer Template (resource group 2)
+
+The Packer template in this resource group accepts [several methods for
+executing custom scripts][pkr]. To pass the exported startup string to it, you
+must collect it from the Terraform module and provide it to the Packer template.
+After running `terraform -chdir=image-builder/builder-env apply` as instructed
+by `ghpc`, execute the following:
+
+```shell
+terraform -chdir=image-builder/builder-env \
+  output -raw startup_script_install_ansible > \
+  image-builder/packer/custom-image/startup_script.sh
+cd image-builder/packer/custom-image
+packer init .
+packer validate -var startup_script_file=startup_script.sh .
+packer build -var startup_script_file=startup_script.sh .
+```
 
 ## Config Schema
 

examples/image-builder.yaml

@@ -20,33 +20,29 @@ vars:
   deployment_name: image-builder-001
   region: us-central1
   zone: us-central1-c
-  ansible_playbooks:
-  - playbook_file: ./example-playbook.yml
-    galaxy_file: null
-    extra_arguments: ["-vv"]
 
 resource_groups:
-- group: network
+- group: builder-env
   resources:
   - source: resources/network/vpc
     kind: terraform
     id: network1
-    settings:
-      primary_subnetwork:
-        name: custom-image-builder-subnetwork
-        description: Custom Image Building Subnetwork
-        new_bits: 15
-        private_access: true
-        flow_logs: false
     outputs:
     - subnetwork_name
+  - source: resources/scripts/startup-script
+    kind: terraform
+    id: install_ansible
+    settings:
+      runners:
+      - type: shell
+        source: modules/startup-script/examples/install_ansible.sh
+        destination: install_ansible.sh
+    outputs:
+    - startup_script
 - group: packer
   resources:
   - source: resources/packer/custom-image
     kind: packer
     id: custom-image
     settings:
-      subnetwork: custom-image-builder-subnetwork
-      use_iap: true
-      omit_external_ip: true
       disk_size: 100

pkg/config/config.go

@@ -49,6 +49,7 @@ var errorMessages = map[string]string{
 	"settingsLabelType": "labels in resources settings are not a map",
 	"invalidVar":        "invalid variable definition in",
 	"varNotFound":       "Could not find source of variable",
+	"varInAnotherGroup": "References to other groups are not yet supported",
 	"noOutput":          "Output not found for a variable",
 	// validator
 	"emptyID":         "a resource id cannot be empty",
@@ -253,20 +254,21 @@ func importYamlConfig(yamlConfigFilename string) YamlConfig {
 }
 
 // ExportYamlConfig exports the internal representation of a blueprint config
-func (bc BlueprintConfig) ExportYamlConfig(outputFilename string) []byte {
+func (bc BlueprintConfig) ExportYamlConfig(outputFilename string) ([]byte, error) {
 	d, err := yaml.Marshal(&bc.Config)
 	if err != nil {
-		log.Fatalf("%s: %v", errorMessages["yamlMarshalError"], err)
+		return d, fmt.Errorf("%s: %w", errorMessages["yamlMarshalError"], err)
 	}
 	if outputFilename == "" {
-		return d
+		return d, nil
 	}
 	err = ioutil.WriteFile(outputFilename, d, 0644)
 	if err != nil {
-		log.Fatalf("%s, Filename: %s",
-			errorMessages["fileSaveError"], outputFilename)
+		// hitting this error writing yaml
+		return d, fmt.Errorf("%s, Filename: %s: %w",
+			errorMessages["fileSaveError"], outputFilename, err)
 	}
-	return nil
+	return nil, nil
 }
 
 func createResourceInfo(
@@ -454,12 +456,10 @@ func ConvertMapToCty(iMap map[string]interface{}) (map[string]cty.Value, error)
 
 // ResolveGlobalVariables given a map of strings to cty.Value types, will examine
 // all cty.Values that are of type cty.String. If they are literal global variables,
-// then they are replaces by the cty.Value of the corresponding entry in yc.Vars
-// the cty.Value types that are string literal global variables to their value
+// then they are replaced by the cty.Value of the corresponding entry in
+// yc.Vars. All other cty.Values are unmodified.
 // ERROR: if conversion from yc.Vars to map[string]cty.Value fails
-// ERROR: if (somehow) the cty.String cannot be covnerted to a Go string
-// ERROR: if there are literal variables which are not globals
-//        (this will be a use case we should consider)
+// ERROR: if (somehow) the cty.String cannot be converted to a Go string
 // ERROR: rely on HCL TraverseAbs to bubble up "diagnostics" when the global variable
 //        being resolved does not exist in yc.Vars
 func (yc *YamlConfig) ResolveGlobalVariables(ctyMap map[string]cty.Value) error {
@@ -477,7 +477,8 @@ func (yc *YamlConfig) ResolveGlobalVariables(ctyMap map[string]cty.Value) error
 				return err
 			}
 			ctx, varName, found := IdentifyLiteralVariable(valString)
-			// confirm literal and that it is global
+			// only attempt resolution on global literal variables
+			// leave all other strings alone (including non-global)
 			if found && ctx == "var" {
 				varTraversal := hcl.Traversal{
 					hcl.TraverseRoot{Name: ctx},
@@ -488,8 +489,6 @@ func (yc *YamlConfig) ResolveGlobalVariables(ctyMap map[string]cty.Value) error
 					return diags
 				}
 				ctyMap[key] = newVal
-			} else {
-				return fmt.Errorf("%s was not a literal global variable ((var.name))", valString)
 			}
 		}
 	}

pkg/config/config_test.go

@@ -340,7 +340,8 @@ func (s *MySuite) TestExportYamlConfig(c *C) {
 	// Return bytes
 	bc := BlueprintConfig{}
 	bc.Config = expectedSimpleYamlConfig
-	obtainedYaml := bc.ExportYamlConfig("")
+	obtainedYaml, err := bc.ExportYamlConfig("")
+	c.Assert(err, IsNil)
 	c.Assert(obtainedYaml, Not(IsNil))
 
 	// Write file
@@ -505,49 +506,51 @@ func (s *MySuite) TestConvertMapToCty(c *C) {
 
 func (s *MySuite) TestResolveGlobalVariables(c *C) {
 	var err error
-	var testkey = "testkey"
+	var testkey1 = "testkey1"
+	var testkey2 = "testkey2"
+	var testkey3 = "testkey3"
 	bc := getBlueprintConfigForTest()
 	ctyMap := make(map[string]cty.Value)
 	err = bc.Config.ResolveGlobalVariables(ctyMap)
 	c.Assert(err, IsNil)
 
-	// confirm that a plain string (non-variable) is unchanged and errors
+	// confirm plain string is unchanged and does not error
 	testCtyString := cty.StringVal("testval")
-	ctyMap[testkey] = testCtyString
+	ctyMap[testkey1] = testCtyString
 	err = bc.Config.ResolveGlobalVariables(ctyMap)
-	c.Assert(err, NotNil)
-	c.Assert(ctyMap[testkey], Equals, testCtyString)
+	c.Assert(err, IsNil)
+	c.Assert(ctyMap[testkey1], Equals, testCtyString)
 
-	// confirm that a literal, but not global, variable is unchanged and errors
+	// confirm literal, non-global, variable is unchanged and does not error
 	testCtyString = cty.StringVal("((module.testval))")
-	ctyMap[testkey] = testCtyString
+	ctyMap[testkey1] = testCtyString
 	err = bc.Config.ResolveGlobalVariables(ctyMap)
-	c.Assert(err, NotNil)
-	c.Assert(ctyMap[testkey], Equals, testCtyString)
+	c.Assert(err, IsNil)
+	c.Assert(ctyMap[testkey1], Equals, testCtyString)
 
 	// confirm failed resolution of a literal global
 	testCtyString = cty.StringVal("((var.test_global_var))")
-	ctyMap[testkey] = testCtyString
+	ctyMap[testkey1] = testCtyString
 	err = bc.Config.ResolveGlobalVariables(ctyMap)
+	c.Assert(err, NotNil)
 	c.Assert(err.Error(), Matches, ".*Unsupported attribute;.*")
 
-	// confirm successful resolution a literal global string
-	testGlobalVar := "test_global_var"
-	testGlobalVarString := "testval"
-	bc.Config.Vars[testGlobalVar] = testGlobalVarString
-	testCtyString = cty.StringVal(fmt.Sprintf("((var.%s))", testGlobalVar))
-	ctyMap[testkey] = testCtyString
-	err = bc.Config.ResolveGlobalVariables(ctyMap)
-	c.Assert(err, IsNil)
-	c.Assert(ctyMap[testkey], Equals, cty.StringVal(testGlobalVarString))
-
-	// confirm successful resolution a literal global boolean
-	testGlobalVar = "test_global_var"
-	testGlobalVarBool := true
-	bc.Config.Vars[testGlobalVar] = testGlobalVarBool
-	testCtyString = cty.StringVal(fmt.Sprintf("((var.%s))", testGlobalVar))
-	ctyMap[testkey] = testCtyString
+	// confirm successful resolution of literal globals in presence of other strings
+	testGlobalVarString := "test_global_string"
+	testGlobalValString := "testval"
+	testGlobalVarBool := "test_global_bool"
+	testGlobalValBool := "testval"
+	testPlainString := "plain-string"
+	bc.Config.Vars[testGlobalVarString] = testGlobalValString
+	bc.Config.Vars[testGlobalVarBool] = testGlobalValBool
+	testCtyString = cty.StringVal(fmt.Sprintf("((var.%s))", testGlobalVarString))
+	testCtyBool := cty.StringVal(fmt.Sprintf("((var.%s))", testGlobalVarBool))
+	ctyMap[testkey1] = testCtyString
+	ctyMap[testkey2] = testCtyBool
+	ctyMap[testkey3] = cty.StringVal(testPlainString)
 	err = bc.Config.ResolveGlobalVariables(ctyMap)
 	c.Assert(err, IsNil)
-	c.Assert(ctyMap[testkey], Equals, cty.BoolVal(testGlobalVarBool))
+	c.Assert(ctyMap[testkey1], Equals, cty.StringVal(testGlobalValString))
+	c.Assert(ctyMap[testkey2], Equals, cty.StringVal(testGlobalValBool))
+	c.Assert(ctyMap[testkey3], Equals, cty.StringVal(testPlainString))
 }

pkg/config/expand.go

@@ -414,7 +414,8 @@ func expandSimpleVariable(
 			errorMessages["varNotFound"], varSource)
 	}
 	if refGrpIndex != context.groupIndex {
-		log.Fatalf("Unimplemented: references to other groups are not yet supported")
+		return "", fmt.Errorf("%s: resource %s was defined in group %d and called from group %d",
+			errorMessages["varInAnotherGroup"], varSource, refGrpIndex, context.groupIndex)
 	}
 
 	// Get the resource info