diff --git a/.gitignore b/.gitignore index ec233e6dbc..09311d90e2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,53 @@ +#### GENERAL DEVELOPMENT + +# Built Binary ghpc +# macOS Desktop Services Store .DS_Store # workspace level vscode settings .vscode/ + +#### TERRAFORM + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +# Exclude all .tfvars files, which are likely to contain sentitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +# +*.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +# Ignore Terraform lock files sometimes generated during validation +# These are useful for root modules, but not re-usable modules +.terraform.lock.hcl + +#### PACKER +packer-manifest.json +*.auto.pkrvars.hcl diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 92aaf076b3..f2663b2759 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,8 +56,8 @@ repos: hooks: - id: go-critic args: [-disable, "#experimental,sloppyTypeAssert"] -- repo: https://github.com/ansible-community/ansible-lint.git - rev: v5.4.0 # https://github.com/ansible-community/ansible-lint/releases/ +- repo: https://github.com/ansible/ansible-lint.git + rev: v5.4.0 hooks: - id: ansible-lint - repo: https://github.com/adrienverge/yamllint.git diff --git a/.tflint.hcl b/.tflint.hcl index 2ba8b4e230..3f9561c7ac 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -13,7 +13,7 @@ // limitations under the License. plugin "google" { enabled = true - version = "0.12.1" + version = "0.16.1" source = "github.com/terraform-linters/tflint-ruleset-google" } rule "terraform_deprecated_index" { diff --git a/README.md b/README.md index da76a80305..4d585f2c40 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,9 @@ You must first set up Cloud Shell to authenticate with GitHub. We will use an SSH key. > **_NOTE:_** You can skip this step if you have previously set up cloud shell -> with GitHub. +> with GitHub.\ > **_NOTE:_** You can find much more detailed instructions for this step in the -> [GitHub docs](https://docs.github.com/en/authentication/connecting-to-github-with-ssh). +> [GitHub docs](https://docs.github.com/en/authentication/connecting-to-github-with-ssh).\ > **_NOTE:_** This step is only required during the private preview of the > HPC-Toolkit. @@ -52,7 +52,7 @@ cd hpc-toolkit && make You should now have a binary named `ghpc` in the project root directory. Optionally, you can run `./ghpc --version` to verify the build. -## Creating HPC Blueprints +## Quick Start To create a blueprint, an input YAML file needs to be written or adapted from one of the [examples](examples/). @@ -74,45 +74,121 @@ These instructions assume you are using you wish to deploy in, and that you are in the root directory of the hpc-toolkit repo cloned during [installation](#installation). -The [examples/hpc-cluster-small.yaml](examples/hpc-cluster-small.yaml) file must -be updated to point to your GCP project ID. You can either edit the file -manually or run the following command. +Run the ghpc binary with the following command: ```shell -sed -i \ - "s/## Set GCP Project ID Here ##/$GOOGLE_CLOUD_PROJECT/g" \ - examples/hpc-cluster-small.yaml +./ghpc create examples/hpc-cluster-small.yaml --vars "project_id=${GOOGLE_CLOUD_PROJECT}" ``` -Now you can run `ghpc` with the following command: +> **_NOTE:_** The `--vars` argument supports comma-separated list of name=value +> variables to override YAML configuration variables. This feature only supports +> variables of string type. -```shell -./ghpc create examples/hpc-cluster-small.yaml -``` +This will create a blueprint directory named `hpc-cluster-small/`. -By default, the blueprint directory will be created in the same directory as the -`ghpc` binary and will have the name specified by the `blueprint_name` field -from the input config. Optionally, the output directory can be specified with -the `-o` flag as shown in the following example. +After successfully running `ghpc create`, a short message displaying how to +proceed is displayed. For the `hpc-cluster-small` example, the message will +appear similar to: ```shell -./ghpc create examples/hpc-cluster-small.yaml -o blueprints/ +terraform -chdir=hpc-cluster-small/primary init +terraform -chdir=hpc-cluster-small/primary apply ``` -## Deploying HPC Blueprints +Use these commands to run terraform and deploy your cluster. If the `apply` is +successful, a message similar to the following will be displayed: -Blueprints are a set of resource groups each composed of Packer templates and -Terraform modules. The process for deploying Terraform modules is documented -below. +```shell +Apply complete! Resources: 13 added, 0 changed, 0 destroyed. +``` > **_NOTE:_** Before you run this for the first time you may need to enable some > APIs and possibly request additional quotas. See > [Enable GCP APIs](#enable-gcp-apis) and -> [Small Example Quotas](examples/README.md#hpc-cluster-smallyaml). +> [Small Example Quotas](examples/README.md#hpc-cluster-smallyaml).\ +> **_NOTE:_** If not using cloud shell you may need to set up +> [GCP Credentials](#gcp-credentials).\ +> **_NOTE:_** Cloud Shell times out after 20 minutes of inactivity. This example +> deploys in about 5 minutes but for more complex deployments it may be +> necessary to deploy (`terraform apply`) from a cloud VM. The same process +> above can be used, although [dependencies](#dependencies) will need to be +> installed first. + +Once the blueprint has successfully been deployed, take the following steps to run a job: + +* First navigate to `Compute Engine` > `VM instances` in the Google Cloud Console. +* Next click on the `SSH` button associated with the `slurm-hpc-small-login0` instance. +* Finally run the `hostname` command on 3 nodes by running the following command in the shell popup: + +```shell +$ srun -N 3 hostname +slurm-hpc-slurm-small-debug-0-0 +slurm-hpc-slurm-small-debug-0-1 +slurm-hpc-slurm-small-debug-0-2 +``` + +By default, this runs the job on the `debug` partition. See details in +[examples/](examples/README.md#compute-partition) for how to run on the more +performant `compute` partition. + +This example does not contain any Packer-based resources but for completeness, +you can use the following command to deploy a Packer-based resource group: + +```shell +cd // +packer build . +``` + +## HPC Toolkit Components + +The HPC Toolkit has been designed to simplify the process of deploying a +familiar HPC cluster on Google Cloud. The block diagram below describes the +individual components of the HPC toolkit. + +```mermaid +graph LR + subgraph Basic Customizations + A(1. GCP-provided reference configs.) --> B(2. Configuration YAML) + end + B --> D + subgraph Advanced Customizations + C(3. Resources, eg. Terraform, Scripts) --> D(4. ghpc Engine) + D --> E(5. Deployment Blueprint) + end + E --> F(6. HPC environment on GCP) +``` + +1. **GCP-provided reference configs** – A set of vetted reference configs can be + found in the examples directory. These can be used to create a predefined + blueprint for a cluster or as a starting point for creating a custom + blueprint. +2. **Configuration YAML** – The primary interface to the HPC Toolkit is an input + YAML file that defines which resources to use and how to customize them. +3. **gHPC Engine** – The gHPC engine converts the configuration YAML into a self-contained blueprint directory. +4. **Resources** – The building blocks of a blueprint directory are the + resources. Resources can be found in the resources directory. They are + composed of terraform, packer and/or script files that meet the expectations + of the gHPC engine. +5. **Deployment Blueprint** – A self-contained directory that can be used to + deploy a cluster onto Google Cloud. This is the output of the gHPC engine. +6. **HPC environment on GCP** – After deployment of a blueprint, an HPC environment will be available in Google Cloud. + +Users can configure a set of resources, and using the gHPC Engine of the HPC +Toolkit, they can produce a blueprint and deployment instructions for creating +those resources. Terraform is the primary method for defining the resources +behind the HPC cluster, but other resources based on tools like ansible and +Packer are available. + +The HPC Toolkit can provide extra flexibility to configure a cluster to the +specifications of a customer by making the blueprints directly available and +editable before deployment. Any HPC customer seeking a quick on-ramp to building +out their infrastructure on GCP can benefit from this. + +## GCP Credentials ### Supplying cloud credentials to Terraform -Terraform can discover credentials for authenticating to Google Clould Platform +Terraform can discover credentials for authenticating to Google Cloud Platform in several ways. We will summarize Terraform's documentation for using [gcloud][terraform-auth-gcloud] from your workstation and for automatically finding credentials in cloud environments. We do **not** recommend following @@ -162,55 +238,66 @@ in particular an inactivity timeout that will close running shells after 20 minutes. Please consider it only for small blueprints that are quickly deployed. -### Running Terraform - -After successfully running `ghpc create`, a short message displaying how to -proceed is displayed. For the `hpc-cluster-small` example, the message will -appear similar to: - -```shell -cd hpc-cluster-small/primary -terraform init -terraform apply +## Blueprint Warnings and Errors + +By default, each blueprint is configured with a number of "validator" functions +which perform basic tests of your global variables. If `project_id`, `region`, +and `zone` are defined as global variables, then the following validators are +enabled: + +```yaml +validators: +- validator: test_project_exists + inputs: + project_id: $(vars.project_id) +- validator: test_region_exists + inputs: + project_id: $(vars.project_id) + region: $(vars.region) +- validator: test_zone_exists + inputs: + project_id: $(vars.project_id) + zone: $(vars.zone) +- validator: test_zone_in_region + inputs: + project_id: $(vars.project_id) + zone: $(vars.zone) + region: $(vars.region) ``` -If the `apply` is successful, a message similar to the following will be -displayed: +This configures validators that check the validity of the project ID, region, +and zone. Additionally, it checks that the zone is in the region. Validators can +be overwritten, however they are limited to the set of functions defined above. -```shell -Apply complete! Resources: 20 added, 0 changed, 0 destroyed. +Validators can be explicitly set to the empty list: + +```yaml +validators: [] ``` -### Testing your cluster -Once the blueprint has successfully been deployed, take the following steps to run a job: +They can also be set to 3 differing levels of behavior using the command-line +`--validation-level` flag` for the `create` and `expand` commands: -* First navigate to `Compute Engine` > `VM instances` in the Google Cloud Console. -* Next click on the `SSH` button associated with the `slurm-hpc-small-login0` instance. -* Finally run the `hostname` command on 3 nodes by running the following command in the shell popup: +* `"ERROR"`: If any validator fails, the blueprint will not be + written. Error messages will be printed to the screen that indicate which + validator(s) failed and how. +* `"WARNING"` (default): The blueprint will be written even if any validators + fail. Warning messages will be printed to the screen that indicate which + validator(s) failed and how. +* `"IGNORE"`: Do not execute any validators, even if they are explicitly + defined in a `validators` block or the default set is implicitly added. + +For example, this command will set all validators to `WARNING` behavior: ```shell -$ srun -N 3 hostname -slurm-hpc-slurm-small-debug-0-0 -slurm-hpc-slurm-small-debug-0-1 -slurm-hpc-slurm-small-debug-0-2 +./ghpc create --validation-level WARNING examples/hpc-cluster-small.yaml ``` -By default, this runs the job on the `debug` partition. See details in -[examples/](examples/README.md#compute-partition) for how to run on the more -performant `compute` partition. - -> **_NOTE:_** Cloud Shell times out after 20 minutes of inactivity. This example -> deploys in about 5 minutes but for more complex deployments it may be -> necessary to deploy (`terraform apply`) from a cloud VM. The same process -> above can be used, although [dependencies](#dependencies) will need to be -> installed first. - -This example does not contain any Packer-based resources but for completeness, -you can use the following command to deploy a Packer-based resource group: +The flag can be shortened to `-l` as shown below using `IGNORE` to disable all +validators. ```shell -cd // -packer build . +./ghpc create -l IGNORE examples/hpc-cluster-small.yaml ``` ## Enable GCP APIs @@ -305,10 +392,53 @@ message. Here are some common reasons for the deployment to fail: [Enable GCP APIs](#enable-gcp-apis). * **Insufficient Quota:** The GCP project does not have enough quota to provision the requested resources. See [GCP Quotas](#gcp-quotas). +* **Filestore resource limit:** When regularly deploying filestore instances + with a new vpc you may see an error during deployment such as: + `System limit for internal resources has been reached`. See + [this doc](https://cloud.google.com/filestore/docs/troubleshooting#api_cannot_be_disabled) + for the solution. +* **Required permission not found:** + * Example: `Required 'compute.projects.get' permission for 'projects/... forbidden` + * Credentials may not be set, or are not set correctly. Please follow + instructions at [Cloud credentials on your workstation](#cloud-credentials-on-your-workstation). + * Ensure proper permissions are set in the cloud console + [IAM section](https://console.cloud.google.com/iam-admin/iam). + +### Failure to Destroy VPC Network + +If `terraform destroy` fails with an error such as the following: + +```text +│ Error: Error when reading or editing Subnetwork: googleapi: Error 400: The subnetwork resource 'projects//regions//subnetworks/' is already being used by 'projects//zones//instances/', resourceInUseByAnotherResource +``` + +or + +```text +│ Error: Error waiting for Deleting Network: The network resource 'projects//global/networks/' is already being used by 'projects//global/firewalls/' +``` + +These errors indicate that the VPC network cannot be destroyed because resources +were added outside of Terraform and that those resources depend upon the +network. These resources should be deleted manually. The first message indicates +that a new VM has been added to a subnetwork within the VPC network. The second +message indicates that a new firewall rule has been added to the VPC network. +If your error message does not look like these, examine it carefully to identify +the type of resouce to delete and its unique name. In the two messages above, +the resource names appear toward the end of the error message. The following +links will take you directly to the areas within the Cloud Console for managing +VMs and Firewall rules. Make certain that your project ID is selected in the +drop-down menu at the top-left. + +* [Cloud Console: Manage VM instances][cc-vms] +* [Cloud Console: Manage Firewall Rules][cc-firewall] + +[cc-vms]: https://console.cloud.google.com/compute/instances +[cc-firewall]: https://console.cloud.google.com/networking/firewalls/list ## Inspecting the Blueprint -The blueprint is created in the directory matching the provided blueprint_name +The blueprint is created in the directory matching the provided blueprint\_name variable in the config. Within this directory are all the resources needed to create a deployment. The blueprint directory will contain subdirectories representing the resource groups defined in the config YAML. Most example @@ -330,6 +460,46 @@ hpc-cluster-small/ vpc/ ``` +## `ghpc` Commands + +### Create + +``` shell +./ghpc create +``` + +The create command is the primary interface for the HPC Toolkit. This command takes the path to a environment definition file as input and creates a blueprint based on it. Further information on creating this config file, see [Writing Config YAML](examples/README.md#writing-config-yaml). + +By default, the blueprint directory will be created in the same directory as the +`ghpc` binary and will have the name specified by the `blueprint_name` field +from the input config. Optionally, the output directory can be specified with +the `-o` flag as shown in the following example. + +```shell +./ghpc create examples/hpc-cluster-small.yaml -o blueprints/ +``` + +### Expand + +```shell +./ghpc expand –out +``` + +The expand command creates an expanded config file with all settings explicitly +listed and variables expanded. This can be a useful tool for creating explicit, +detailed examples and for debugging purposes. The expanded yaml is still valid +as input to [`ghpc create`](#create) to create the blueprint. + +### Completion + +```shell +./ghpc completion [bash|zsh|fish|powershell] +``` + +The completion command creates a shell completion config file for the specified shell. To apply the configuration file created by the command, it is required to set up for each shell. For example, loading the completion config by .bashrc is required for Bash. + +Call `ghpc completion --help` for shell specific setup instructions. + ## Dependencies Much of the HPC Toolkit blueprint is built using Terraform and Packer, and @@ -368,6 +538,11 @@ successfully passing. 1. Install pre-commit using the instructions from [the pre-commit website](https://pre-commit.com/). 1. Install TFLint using the instructions from [the TFLint documentation](https://github.com/terraform-linters/tflint#installation). + * Note: The version of TFLint must be compatible with the Google plugin + version identified in [tflint.hcl](.tflint.hcl). Versions of the plugin + `>=0.16.0` should use `tflint>=0.35.0` and versions of the plugin + `<=0.15.0` should preferably use `tflint==0.34.1`. These versions are + readily available via GitHub or package managers. 1. Install ShellCheck using the instructions from [the ShellCheck documentation](https://github.com/koalaman/shellcheck#installing) 1. The other dev dependencies can be installed by running the following command @@ -398,3 +573,10 @@ the following script against the packer config file: ```shell tools/autodoc/terraform_docs.sh resources/packer/new_resource/image.json ``` + +### Contributing + +Please refer to the [contributing file](CONTRIBUTING.md) in our github repo, or +to +[Google’s Open Source documentation](https://opensource.google/docs/releasing/template/CONTRIBUTING/#). +Before submitting, we recommend contributors run pre-commit tests (more below). diff --git a/cmd/create.go b/cmd/create.go index 1bbea16c6f..938bbff0f6 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -21,10 +21,13 @@ import ( "fmt" "hpc-toolkit/pkg/config" "hpc-toolkit/pkg/reswriter" + "log" "github.com/spf13/cobra" ) +const msgCLIVars = "Comma-separated list of name=value variables to override YAML configuration. Can be invoked multiple times." + func init() { createCmd.Flags().StringVarP(&yamlFilename, "config", "c", "", "Configuration file for the new blueprints") @@ -32,13 +35,19 @@ func init() { "please see the command usage for more details.")) createCmd.Flags().StringVarP(&bpDirectory, "out", "o", "", "Output directory for the new blueprints") + createCmd.Flags().StringSliceVar(&cliVariables, "vars", nil, msgCLIVars) + createCmd.Flags().StringVarP(&validationLevel, "validation-level", "l", "WARNING", + validationLevelDesc) rootCmd.AddCommand(createCmd) } var ( - yamlFilename string - bpDirectory string - createCmd = &cobra.Command{ + yamlFilename string + bpDirectory string + cliVariables []string + validationLevel string + validationLevelDesc = "Set validation level to one of (\"ERROR\", \"WARNING\", \"IGNORE\")" + createCmd = &cobra.Command{ Use: "create FILENAME", Short: "Create a new blueprint.", Long: "Create a new blueprint based on a provided YAML config.", @@ -57,6 +66,14 @@ func runCreateCmd(cmd *cobra.Command, args []string) { } blueprintConfig := config.NewBlueprintConfig(yamlFilename) + if err := blueprintConfig.SetCLIVariables(cliVariables); err != nil { + log.Fatalf("Failed to set the variables at CLI: %v", err) + } + if err := blueprintConfig.SetValidationLevel(validationLevel); err != nil { + log.Fatal(err) + } blueprintConfig.ExpandConfig() - reswriter.WriteBlueprint(&blueprintConfig.Config, bpDirectory) + if err := reswriter.WriteBlueprint(&blueprintConfig.Config, bpDirectory); err != nil { + log.Fatal(err) + } } diff --git a/cmd/expand.go b/cmd/expand.go index 4e5c644f88..3180cc4589 100644 --- a/cmd/expand.go +++ b/cmd/expand.go @@ -18,6 +18,7 @@ package cmd import ( "fmt" "hpc-toolkit/pkg/config" + "log" "github.com/spf13/cobra" ) @@ -29,6 +30,9 @@ func init() { "please see the command usage for more details.")) expandCmd.Flags().StringVarP(&outputFilename, "out", "o", "expanded.yaml", "Output file for the expanded yaml.") + expandCmd.Flags().StringSliceVar(&cliVariables, "vars", nil, msgCLIVars) + expandCmd.Flags().StringVarP(&validationLevel, "validation-level", "l", "WARNING", + validationLevelDesc) rootCmd.AddCommand(expandCmd) } @@ -53,6 +57,12 @@ func runExpandCmd(cmd *cobra.Command, args []string) { } blueprintConfig := config.NewBlueprintConfig(yamlFilename) + if err := blueprintConfig.SetCLIVariables(cliVariables); err != nil { + log.Fatalf("Failed to set the variables at CLI: %v", err) + } + if err := blueprintConfig.SetValidationLevel(validationLevel); err != nil { + log.Fatal(err) + } blueprintConfig.ExpandConfig() blueprintConfig.ExportYamlConfig(outputFilename) fmt.Printf( diff --git a/cmd/root.go b/cmd/root.go index 61c12f881f..64a49fc2ff 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -34,7 +34,7 @@ HPC deployments on the Google Cloud Platform.`, log.Fatalf("cmd.Help function failed: %s", err) } }, - Version: "v0.5.0-alpha (private preview)", + Version: "v0.6.0-alpha (private preview)", } ) diff --git a/examples/README.md b/examples/README.md index 641308e280..fe26204a9f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -103,16 +103,45 @@ Quota required for this example: ### spack-gromacs.yaml -Spack is a HPC software package manager. This example creates a -[Spack](../resources/scripts/spack-install/README.md) build VM and a -workstation for testing and validating a spack build. The build VM will install -and configure spack, and install gromacs with spack (as configured in the -spack-install resource). This happens in a shared location (/apps). Then the -build VM will shutdown. This build leverages the startup-script resource and -can be applied in any cluster by using the output of spack-install or -startup-script resources. - -Note: Installing spack compilers and libraries in this example can take 1-2 +Spack is a HPC software package manager. This example creates a small slurm +cluster with software installed with +[Spack](../resources/scripts/spack-install/README.md) The controller will +install and configure spack, and install [gromacs](https://www.gromacs.org/) +using spack. Spack is installed in a shared location (/apps) via filestore. This +build leverages the startup-script resource and can be applied in any cluster by +using the output of spack-install or startup-script resources. + +The installation will occur as part of the slurm startup-script, a warning +message will be displayed upon SSHing to the login node indicating +that configuration is still active. To track the status of the overall +startup script, run the following command on the login node: + +```shell +sudo tail -f /var/log/messages +``` + +Spack specific installation logs will be sent to the spack_log as configured in +your YAML, by default /var/log/spack.log in the login node. + +```shell +sudo tail -f /var/log/spack.log +``` + +Once Slurm and spack installation is complete, spack will available on the login +node. To use spack in the controller or compute nodes, the following command +must be run first: + +```shell +source /apps/spack/share/spack/setup-env.sh +``` + +To load the gromacs module, use spack: + +```shell +spack load gromacs +``` + + **_NOTE:_** Installing spack compilers and libraries in this example can take 1-2 hours to run on startup. To decrease this time in future deployments, consider including a spack build cache as described in the comments of the example. @@ -123,6 +152,50 @@ omnia-manager node and 2 omnia-compute nodes, on the pre-existing default network. Omnia will be automatically installed after the nodes are provisioned. All nodes mount a filestore instance on `/home`. +### image-builder.yaml + +This Blueprint helps create custom VM images by applying necessary software and +configurations to existing images, such as the [HPC VM Image][hpcimage]. +Using a custom VM image can be more scalable than installing software using +boot-time startup scripts because + +* it avoids reliance on continued availability of package repositories +* VMs will join an HPC cluster and execute workloads more rapidly due to reduced + boot-time configuration +* machines are guaranteed to boot with a static set of packages available when + the custom image was created. No potential for some machines to be upgraded + relative to other based upon their creation time! + +[hpcimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm + +**Note**: this example relies on the default behavior of the Toolkit to derive +naming convention for networks and other resources from the `deployment_name`. + +#### Custom Network (resource group) + +A tool called [Packer](https://packer.io) builds custom VM images by creating +short-lived VMs, executing scripts on them, and saving the boot disk as an +image that can be used by future VMs. The short-lived VM must operate in a +network that + +* has outbound access to the internet for downloading software +* has SSH access from the machine running Packer so that local files/scripts + can be copied to the VM + +This resource group creates such a network, while using [Cloud Nat][cloudnat] +and [Identity-Aware Proxy (IAP)][iap] to allow outbound traffic and inbound SSH +connections without exposing the machine to the internet on a public IP address. + +[cloudnat]: https://cloud.google.com/nat/docs/overview +[iap]: https://cloud.google.com/iap/docs/using-tcp-forwarding + +#### Packer Template (resource group) + +The Packer template in this resource group accepts a list of Ansible playbooks +which will be run on the VM to customize it. Although it defaults to creating +VMs with a public IP address, it can be easily set to use [IAP][iap] for SSH +tunneling following the [example in its README](../resources/packer/custom-image/README.md). + ## Config Schema A user defined config should follow the following schema: @@ -178,6 +251,80 @@ resource_groups: - source: github.com/org/repo//resources/role/resource-name ``` +## Writing Config YAML + +The input YAML is composed of 3 primary parts, top-level parameters, global variables and resources group. These are described in more detail below. + +### Top Level Parameters + +* **blueprint_name** (required): Name of this set of blueprints. This also defines the name of the directory the blueprints will be created into. + +### Global Variables + +```yaml +vars: + region: "us-west-1" + labels: + "user-defined-global-label": "slurm-cluster" + ... +``` + +Global variables are set under the vars field at the top level of the YAML. +These variables can be explicitly referenced in resources as +[Config Variables](#config-variables). Any resource setting (inputs) not explicitly provided and +matching exactly a global variable name will automatically be set to these +values. + +Global variables should be used with care. Resource default settings with the +same name as a global variable and not explicitly set will be overwritten by the +global variable. + +The global “labels” variable is a special case as it will be appended to labels +found in resource settings, whereas normally an explicit resource setting would +be left unchanged. This ensures that global labels can be set alongside resource +specific labels. Precedence is given to the resource specific labels if a +collision occurs. Default resource labels will still be overwritten by global +labels. + +The HPC Toolkit uses special reserved labels for monitoring each deployment. +These are set automatically, but can be overridden through global vars or +resource settings. They include: + +* ghpc_blueprint: The name of the blueprint the deployment was created from +* ghpc_deployment: The name of the specific deployment of the blueprint +* ghpc_role: The role of a given resource, e.g. compute, network, or + file-system. By default, it will be taken from the folder immediately + containing the resource. Example: A resource with the source path of + `./resources/network/vpc` will have `network` as its `ghpc_role` label by + default. + +### Resource Groups + +Resource groups allow distinct sets of resources to be defined and deployed as a +group. A resource group can only contain resources of a single kind, for example +a resource group may not mix packer and terraform resources. + +For terraform resources, a top-level main.tf will be created for each resource +group so different groups can be created or destroyed independently. + +A resource group is made of 2 fields, group and resources. They are described in +more detail below. + +#### Group + +Defines the name of the group. Each group must have a unique name. The name will +be used to create the subdirectory in the blueprint directory that the resource +group will be defined in. + +#### Resources + +Resources are the building blocks of an HPC environment. They can be composed to +create complex deployments using the config YAML. Several resources are provided +by default in the [resources](../resources/README.md) folder. + +To learn more about how to refer to a resource in a YAML, please consult the +[resources README file.](../resources/README.md) + ## Variables Variables can be used to refer both to values defined elsewhere in the config @@ -244,12 +391,3 @@ everything inside will be provided as is to the resource. Whenever possible, config variables are preferred over literal variables. `ghpc` will perform basic validation making sure all config variables are defined before creating a blueprint making debugging quicker and easier. - -## Resources - -Resources are the building blocks of an HPC environment. They can be composed to -create complex deployments using the config YAML. Several resources are provided -by default in the [resources](../resources/README.md) folder. - -To learn more about how to refer to a resource in a YAML, please consult the -[resources README file.](../resources/README.md) diff --git a/examples/hpc-cluster-high-io.yaml b/examples/hpc-cluster-high-io.yaml index 67e8fe3c8c..08b525c68e 100644 --- a/examples/hpc-cluster-high-io.yaml +++ b/examples/hpc-cluster-high-io.yaml @@ -67,6 +67,7 @@ resource_groups: partition_name: low_cost max_node_count: 10 enable_placement: false + exclusive: false machine_type: n2-standard-4 # This compute_partition is far more performant than low_cost_partition. @@ -104,3 +105,8 @@ resource_groups: - slurm_controller settings: login_machine_type: n2-standard-4 + + - source: resources/monitoring/dashboard + kind: terraform + id: hpc_dashboard + outputs: [instructions] diff --git a/examples/hpc-cluster-small.yaml b/examples/hpc-cluster-small.yaml index b79cc87dbf..38ad798529 100644 --- a/examples/hpc-cluster-small.yaml +++ b/examples/hpc-cluster-small.yaml @@ -50,6 +50,7 @@ resource_groups: partition_name: debug max_node_count: 4 enable_placement: false + exclusive: false machine_type: n2-standard-2 # This compute_partition is far more performant than debug_partition but may require requesting GCP quotas first. diff --git a/examples/image-builder.yaml b/examples/image-builder.yaml new file mode 100644 index 0000000000..6802e3fae2 --- /dev/null +++ b/examples/image-builder.yaml @@ -0,0 +1,44 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +blueprint_name: image-builder + +vars: + project_id: ## Set GCP Project ID Here ## + deployment_name: image-builder-001 + region: us-central1 + zone: us-central1-c + +resource_groups: +- group: network + resources: + - source: resources/network/vpc + kind: terraform + id: network1 + outputs: + - subnetwork_name +- group: packer + resources: + - source: resources/packer/custom-image + kind: packer + id: custom-image + settings: + use_iap: true + omit_external_ip: true + disk_size: 100 + ansible_playbooks: + - playbook_file: ./example-playbook.yml + galaxy_file: ./requirements.yml + extra_arguments: ["-vv"] diff --git a/examples/omnia-cluster.yaml b/examples/omnia-cluster.yaml index 9f02961577..7e5a094ecf 100644 --- a/examples/omnia-cluster.yaml +++ b/examples/omnia-cluster.yaml @@ -19,7 +19,7 @@ blueprint_name: omnia-cluster vars: project_id: ## Set GCP Project ID Here ## deployment_name: omnia-cluster - zone: us-central1-a + zone: us-central1-c region: us-central1 resource_groups: @@ -96,3 +96,12 @@ resource_groups: settings: name_prefix: omnia-compute instance_count: 2 + + # This resource simply makes terraform wait until the startup script is complete + - source: resources/scripts/wait-for-startup + kind: terraform + id: wait + use: + - network + settings: + instance_name: ((module.manager.name[0])) diff --git a/examples/spack-gromacs.yaml b/examples/spack-gromacs.yaml index e4fcfccc49..d3e8bade65 100644 --- a/examples/spack-gromacs.yaml +++ b/examples/spack-gromacs.yaml @@ -14,13 +14,13 @@ --- -blueprint_name: spack-build +blueprint_name: spack-gromacs vars: project_id: ## Set GCP Project ID Here ## - deployment_name: spack + deployment_name: spack-gromacs region: us-central1 - zone: us-central1-a + zone: us-central1-c resource_groups: - group: primary @@ -29,40 +29,43 @@ resource_groups: kind: terraform id: network1 + ## Filesystems - source: resources/file-system/filestore kind: terraform id: appsfs use: [network1] settings: - local_mount: /apps + local_mount: /sw + - source: resources/file-system/filestore + kind: terraform + id: homefs + use: [network1] + settings: + local_mount: /home + + ## Install Scripts - source: resources/scripts/spack-install kind: terraform id: spack settings: - install_dir: /apps/spack + install_dir: /sw/spack spack_url: https://github.com/spack/spack spack_ref: v0.17.1 log_file: /var/log/spack.log configs: + - type: single-config + scope: defaults + value: "config:build_stage:/apps/spack/spack-stage" - type: file - scope: site + scope: defaults value: | modules: tcl: hash_length: 0 - whitelist: - - gcc - blacklist: - - '%gcc@4.8.5' all: conflict: - '{name}' - filter: - environment_blacklist: - - "C_INCLUDE_PATH" - - "CPLUS_INCLUDE_PATH" - - "LIBRARY_PATH" projections: all: '{name}/{version}-{compiler.name}-{compiler.version}' compilers: @@ -73,8 +76,8 @@ resource_groups: # Uncomment and update the name and path to add a shared or personal Spack # cache location to speed up future deployments. # spack_cache_url: - # - mirror_name: gcs_cache - # mirror_url: gs://bucket-name/... + # - mirror_name: gcs_cache + # mirror_url: gs://bucket-name/... - source: resources/scripts/startup-script kind: terraform @@ -84,55 +87,38 @@ resource_groups: - type: shell source: modules/startup-script/examples/install_ansible.sh destination: install_ansible.sh - - type: shell - content: $(appsfs.install_nfs_client) - destination: install-nfs.sh - - type: ansible-local - source: modules/startup-script/examples/mount.yaml - destination: "mount.yaml" - - type: ansible-local - source: modules/spack-install/scripts/install_spack_deps.yml - destination: install_spack_deps.yml - - type: shell - content: $(spack.startup_script) - destination: install_spack.sh - - type: shell - destination: shutdown.sh - content: shutdown -h + - $(spack.install_spack_deps_runner) + - $(spack.install_spack_runner) - - source: resources/compute/simple-instance + - source: resources/third-party/compute/SchedMD-slurm-on-gcp-partition kind: terraform - id: spack-build + id: compute_partition use: - network1 + - homefs - appsfs - - spack-startup settings: - name_prefix: spack-builder - machine_type: n2-standard-8 + partition_name: compute + max_node_count: 20 - - source: resources/scripts/startup-script + - source: resources/third-party/scheduler/SchedMD-slurm-on-gcp-controller kind: terraform - id: mount-startup + id: slurm_controller + use: + - network1 + - homefs + - appsfs + - compute_partition settings: - runners: - - type: shell - source: modules/startup-script/examples/install_ansible.sh - destination: install_ansible.sh - - type: shell - content: $(appsfs.install_nfs_client) - destination: install-nfs.sh - - type: ansible-local - source: modules/startup-script/examples/mount.yaml - destination: "mount.yaml" + login_node_count: 1 - - source: resources/compute/simple-instance + - source: resources/third-party/scheduler/SchedMD-slurm-on-gcp-login-node kind: terraform - id: workstation + id: slurm_login use: - network1 + - homefs - appsfs - - mount-startup + - slurm_controller settings: - name_prefix: workstation - machine_type: n2-standard-8 + login_startup_script: $(spack-startup.startup_script) diff --git a/go.mod b/go.mod index 68a1c0ef26..6e719a19ae 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,9 @@ module hpc-toolkit go 1.16 require ( - github.com/google/go-cmp v0.5.6 // indirect + cloud.google.com/go/compute v1.5.0 + cloud.google.com/go/iam v0.3.0 // indirect + cloud.google.com/go/resourcemanager v1.2.0 github.com/hashicorp/go-getter v1.5.11 github.com/hashicorp/hcl/v2 v2.10.1 github.com/hashicorp/terraform-config-inspect v0.0.0-20210625153042-09f34846faab @@ -15,7 +17,7 @@ require ( github.com/spf13/afero v1.6.0 github.com/spf13/cobra v1.2.1 github.com/zclconf/go-cty v1.9.1 - golang.org/x/text v0.3.6 // indirect + google.golang.org/genproto v0.0.0-20220323144105-ec3c684e5b14 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 2f92f6dfb1..af24c1d87c 100644 --- a/go.sum +++ b/go.sum @@ -17,21 +17,38 @@ cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKP cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.2 h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0 h1:b1zWmYuuHz7gO9kDcM/EpHGr06UgsYNRpNJzI2kFiLM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/resourcemanager v1.2.0 h1:Oyt8+J80B51HgIPNk3p1ezTamu1wVj2bj7rBwL5Qd6k= +cloud.google.com/go/resourcemanager v1.2.0/go.mod h1:hFYbG0p7E8vVfQO3yfeaqEQVFO6n9gg9W2czYIdSEy4= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -41,6 +58,7 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXvaqE= github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= @@ -60,6 +78,8 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -68,6 +88,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -81,6 +106,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -105,6 +132,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -124,6 +152,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -137,14 +166,16 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -156,11 +187,16 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= @@ -206,7 +242,6 @@ github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Ao github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -275,6 +310,7 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -322,6 +358,7 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= @@ -356,7 +393,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -368,7 +404,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -406,8 +441,10 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -419,8 +456,13 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 h1:0Ja1LBD+yisY6RWM/BH7TJVXWsSjs2VwBSmvSX4HdBc= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -473,9 +515,24 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -483,8 +540,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -538,8 +596,11 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -566,8 +627,20 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0 h1:URs6qR1lAxDsqWITsQXI4ZkGiYJ5dHtRNiCpfs2OeKA= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0 h1:67zQnAE0T2rB0A3CwLSas0K+SbVzSxP+zTLkQLexeiw= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -616,8 +689,32 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220323144105-ec3c684e5b14 h1:17TOyVD+9MLIDtDJW9PdtMuVT7gNLEkN+G/xFYjZmr8= +google.golang.org/genproto v0.0.0-20220323144105-ec3c684e5b14/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -637,8 +734,17 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -650,8 +756,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= diff --git a/pkg/config/config.go b/pkg/config/config.go index f47738272d..3273dc1cd4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -16,10 +16,17 @@ package config import ( + "encoding/json" "fmt" "io/ioutil" "log" + "regexp" + "strings" + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/gocty" + ctyJson "github.com/zclconf/go-cty/cty/json" "gopkg.in/yaml.v2" "hpc-toolkit/pkg/resreader" @@ -42,6 +49,7 @@ var errorMessages = map[string]string{ "settingsLabelType": "labels in resources settings are not a map", "invalidVar": "invalid variable definition in", "varNotFound": "Could not find source of variable", + "varInAnotherGroup": "References to other groups are not yet supported", "noOutput": "Output not found for a variable", // validator "emptyID": "a resource id cannot be empty", @@ -79,6 +87,65 @@ type TerraformBackend struct { Configuration map[string]interface{} } +type validatorName int64 + +const ( + // Undefined will be default and potentially throw errors if used + Undefined validatorName = iota + testProjectExistsName + testRegionExistsName + testZoneExistsName + testZoneInRegionName +) + +// this enum will be used to control how fatal validator failures will be +// treated during blueprint creation +const ( + validationError int = iota + validationWarning + validationIgnore +) + +func isValidValidationLevel(level int) bool { + return !(level > validationIgnore || level < validationError) +} + +// SetValidationLevel allows command-line tools to set the validation level +func (bc *BlueprintConfig) SetValidationLevel(level string) error { + switch level { + case "ERROR": + bc.Config.ValidationLevel = validationError + case "WARNING": + bc.Config.ValidationLevel = validationWarning + case "IGNORE": + bc.Config.ValidationLevel = validationIgnore + default: + return fmt.Errorf("invalid validation level (\"ERROR\", \"WARNING\", \"IGNORE\")") + } + + return nil +} + +func (v validatorName) String() string { + switch v { + case testProjectExistsName: + return "test_project_exists" + case testRegionExistsName: + return "test_region_exists" + case testZoneExistsName: + return "test_zone_exists" + case testZoneInRegionName: + return "test_zone_in_region" + default: + return "unknown_validator" + } +} + +type validatorConfig struct { + Validator string + Inputs map[string]interface{} +} + // HasKind checks to see if a resource group contains any resources of the given // kind. Note that a resourceGroup should never have more than one kind, this // function is used in the validation step to ensure that is true. @@ -103,17 +170,6 @@ type Resource struct { Settings map[string]interface{} } -// getSetSettings returns a slice of explicitly set settings at a given point. -func (r Resource) getSetSettings() []string { - setSettings := make([]string, len(r.Settings)) - i := 0 - for setting := range r.Settings { - setSettings[i] = setting - i++ - } - return setSettings -} - // createWrapSettingsWith ensures WrapSettingsWith field is not nil, if it is // a new map is created. func (r *Resource) createWrapSettingsWith() { @@ -123,8 +179,13 @@ func (r *Resource) createWrapSettingsWith() { } // YamlConfig stores the contents on the User YAML +// omitempty on validation_level ensures that expand will not expose the setting +// unless it has been set to a non-default value; the implementation as an +// integer is primarily for internal purposes even if it can be set in blueprint type YamlConfig struct { BlueprintName string `yaml:"blueprint_name"` + Validators []validatorConfig + ValidationLevel int `yaml:"validation_level,omitempty"` Vars map[string]interface{} ResourceGroups []ResourceGroup `yaml:"resource_groups"` TerraformBackendDefaults TerraformBackend `yaml:"terraform_backend_defaults"` @@ -179,6 +240,16 @@ func importYamlConfig(yamlConfigFilename string) YamlConfig { yamlConfig.Vars = make(map[string]interface{}) } + if len(yamlConfig.Vars) == 0 { + yamlConfig.Vars = make(map[string]interface{}) + } + + // if the validation level has been explicitly set to an invalid value + // in YAML blueprint then silently default to validationError + if !isValidValidationLevel(yamlConfig.ValidationLevel) { + yamlConfig.ValidationLevel = validationError + } + return yamlConfig } @@ -304,3 +375,121 @@ func (bc *BlueprintConfig) validateConfig() { log.Fatal(err) } } + +// SetCLIVariables sets the variables at CLI +func (bc *BlueprintConfig) SetCLIVariables(cliVariables []string) error { + for _, cliVar := range cliVariables { + arr := strings.SplitN(cliVar, "=", 2) + + if len(arr) != 2 { + return fmt.Errorf("invalid format: '%s' should follow the 'name=value' format", cliVar) + } + + key, value := arr[0], arr[1] + bc.Config.Vars[key] = value + } + + return nil +} + +// IsLiteralVariable returns true if string matches variable ((ctx.name)) +func IsLiteralVariable(str string) bool { + match, err := regexp.MatchString(literalExp, str) + if err != nil { + log.Fatalf("Failed checking if variable is a literal: %v", err) + } + return match +} + +// IdentifyLiteralVariable returns +// string: variable source (e.g. global "vars" or module "modname") +// string: variable name (e.g. "project_id") +// bool: true/false reflecting success +func IdentifyLiteralVariable(str string) (string, string, bool) { + re := regexp.MustCompile(literalSplitExp) + contents := re.FindStringSubmatch(str) + if len(contents) != 3 { + return "", "", false + } + + return contents[1], contents[2], true +} + +// HandleLiteralVariable is exported for use in reswriter as well +func HandleLiteralVariable(str string) string { + re := regexp.MustCompile(literalExp) + contents := re.FindStringSubmatch(str) + if len(contents) != 2 { + log.Fatalf("Incorrectly formatted literal variable: %s", str) + } + + return strings.TrimSpace(contents[1]) +} + +// ConvertToCty convert interface directly to a cty.Value +func ConvertToCty(val interface{}) (cty.Value, error) { + // Convert to JSON bytes + jsonBytes, err := json.Marshal(val) + if err != nil { + return cty.Value{}, err + } + + // Unmarshal JSON into cty + simpleJSON := ctyJson.SimpleJSONValue{} + simpleJSON.UnmarshalJSON(jsonBytes) + return simpleJSON.Value, nil +} + +// ConvertMapToCty convert an interface map to a map of cty.Values +func ConvertMapToCty(iMap map[string]interface{}) (map[string]cty.Value, error) { + cMap := make(map[string]cty.Value) + for k, v := range iMap { + convertedVal, err := ConvertToCty(v) + if err != nil { + return cMap, err + } + cMap[k] = convertedVal + } + return cMap, nil +} + +// ResolveGlobalVariables given a map of strings to cty.Value types, will examine +// all cty.Values that are of type cty.String. If they are literal global variables, +// then they are replaced by the cty.Value of the corresponding entry in +// yc.Vars. All other cty.Values are unmodified. +// ERROR: if conversion from yc.Vars to map[string]cty.Value fails +// ERROR: if (somehow) the cty.String cannot be converted to a Go string +// ERROR: rely on HCL TraverseAbs to bubble up "diagnostics" when the global variable +// being resolved does not exist in yc.Vars +func (yc *YamlConfig) ResolveGlobalVariables(ctyMap map[string]cty.Value) error { + ctyVars, err := ConvertMapToCty(yc.Vars) + if err != nil { + return fmt.Errorf("could not convert global variables to cty map") + } + evalCtx := &hcl.EvalContext{ + Variables: map[string]cty.Value{"var": cty.ObjectVal(ctyVars)}, + } + for key, val := range ctyMap { + if val.Type() == cty.String { + var valString string + if err := gocty.FromCtyValue(val, &valString); err != nil { + return err + } + ctx, varName, found := IdentifyLiteralVariable(valString) + // only attempt resolution on global literal variables + // leave all other strings alone (including non-global) + if found && ctx == "var" { + varTraversal := hcl.Traversal{ + hcl.TraverseRoot{Name: ctx}, + hcl.TraverseAttr{Name: varName}, + } + newVal, diags := varTraversal.TraverseAbs(evalCtx) + if diags.HasErrors() { + return diags + } + ctyMap[key] = newVal + } + } + } + return nil +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 9f3f49bd19..330b1a221e 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -17,6 +17,7 @@ limitations under the License. package config import ( + "fmt" "io/ioutil" "log" "os" @@ -26,6 +27,7 @@ import ( "hpc-toolkit/pkg/resreader" + "github.com/zclconf/go-cty/cty" . "gopkg.in/check.v1" ) @@ -185,6 +187,7 @@ func getBlueprintConfigForTest() BlueprintConfig { } testYamlConfig := YamlConfig{ BlueprintName: "simple", + Validators: []validatorConfig{}, Vars: map[string]interface{}{}, TerraformBackendDefaults: TerraformBackend{ Type: "", @@ -351,9 +354,202 @@ func (s *MySuite) TestExportYamlConfig(c *C) { c.Assert(fileInfo.IsDir(), Equals, false) } +func (s *MySuite) TestSetCLIVariables(c *C) { + // Success + bc := getBasicBlueprintConfigWithTestResource() + c.Assert(bc.Config.Vars["project_id"], IsNil) + c.Assert(bc.Config.Vars["deployment_name"], IsNil) + c.Assert(bc.Config.Vars["region"], IsNil) + c.Assert(bc.Config.Vars["zone"], IsNil) + + cliProjectID := "cli_test_project_id" + cliDeploymentName := "cli_deployment_name" + cliRegion := "cli_region" + cliZone := "cli_zone" + cliKeyVal := "key=val" + cliVars := []string{ + fmt.Sprintf("project_id=%s", cliProjectID), + fmt.Sprintf("deployment_name=%s", cliDeploymentName), + fmt.Sprintf("region=%s", cliRegion), + fmt.Sprintf("zone=%s", cliZone), + fmt.Sprintf("kv=%s", cliKeyVal), + } + err := bc.SetCLIVariables(cliVars) + + c.Assert(err, IsNil) + c.Assert(bc.Config.Vars["project_id"], Equals, cliProjectID) + c.Assert(bc.Config.Vars["deployment_name"], Equals, cliDeploymentName) + c.Assert(bc.Config.Vars["region"], Equals, cliRegion) + c.Assert(bc.Config.Vars["zone"], Equals, cliZone) + c.Assert(bc.Config.Vars["kv"], Equals, cliKeyVal) + + // Failure: Variable without '=' + bc = getBasicBlueprintConfigWithTestResource() + c.Assert(bc.Config.Vars["project_id"], IsNil) + + invalidNonEQVars := []string{ + fmt.Sprintf("project_id%s", cliProjectID), + } + err = bc.SetCLIVariables(invalidNonEQVars) + + expErr := "invalid format: .*" + c.Assert(err, ErrorMatches, expErr) + c.Assert(bc.Config.Vars["project_id"], IsNil) +} + func TestMain(m *testing.M) { setup() code := m.Run() teardown() os.Exit(code) } + +func (s *MySuite) TestValidationLevels(c *C) { + var err error + var ok bool + bc := getBlueprintConfigForTest() + validLevels := []string{"ERROR", "WARNING", "IGNORE"} + for idx, level := range validLevels { + err = bc.SetValidationLevel(level) + c.Assert(err, IsNil) + ok = isValidValidationLevel(idx) + c.Assert(ok, Equals, true) + } + + err = bc.SetValidationLevel("INVALID") + c.Assert(err, NotNil) + + // check that our test for iota enum is working + ok = isValidValidationLevel(-1) + c.Assert(ok, Equals, false) + invalidLevel := len(validLevels) + 1 + ok = isValidValidationLevel(invalidLevel) + c.Assert(ok, Equals, false) +} + +func (s *MySuite) TestIsLiteralVariable(c *C) { + var matched bool + matched = IsLiteralVariable("((var.project_id))") + c.Assert(matched, Equals, true) + matched = IsLiteralVariable("(( var.project_id ))") + c.Assert(matched, Equals, true) + matched = IsLiteralVariable("(var.project_id)") + c.Assert(matched, Equals, false) + matched = IsLiteralVariable("var.project_id") + c.Assert(matched, Equals, false) +} + +func (s *MySuite) TestIdentifyLiteralVariable(c *C) { + var ctx, name string + var ok bool + ctx, name, ok = IdentifyLiteralVariable("((var.project_id))") + c.Assert(ctx, Equals, "var") + c.Assert(name, Equals, "project_id") + c.Assert(ok, Equals, true) + + ctx, name, ok = IdentifyLiteralVariable("((module.structure.nested_value))") + c.Assert(ctx, Equals, "module") + c.Assert(name, Equals, "structure.nested_value") + c.Assert(ok, Equals, true) + + // TODO: properly variables with periods in them! + // One purpose of literal variables is to refer to values in nested + // structures of a module output; should probably accept that case + // but not global variables with periods in them + ctx, name, ok = IdentifyLiteralVariable("var.project_id") + c.Assert(ctx, Equals, "") + c.Assert(name, Equals, "") + c.Assert(ok, Equals, false) +} + +func (s *MySuite) TestConvertToCty(c *C) { + var testval interface{} + var testcty cty.Value + var err error + + testval = "test" + testcty, err = ConvertToCty(testval) + c.Assert(testcty.Type(), Equals, cty.String) + c.Assert(err, IsNil) + + testval = complex(1, -1) + testcty, err = ConvertToCty(testval) + c.Assert(testcty.Type(), Equals, cty.NilType) + c.Assert(err, NotNil) +} + +func (s *MySuite) TestConvertMapToCty(c *C) { + var testmap map[string]interface{} + var testcty map[string]cty.Value + var err error + var testkey = "testkey" + var testval = "testval" + testmap = map[string]interface{}{ + testkey: testval, + } + + testcty, err = ConvertMapToCty(testmap) + c.Assert(err, IsNil) + ctyval, found := testcty[testkey] + c.Assert(found, Equals, true) + c.Assert(ctyval.Type(), Equals, cty.String) + + testmap = map[string]interface{}{ + "testkey": complex(1, -1), + } + testcty, err = ConvertMapToCty(testmap) + c.Assert(err, NotNil) + ctyval, found = testcty[testkey] + c.Assert(found, Equals, false) +} + +func (s *MySuite) TestResolveGlobalVariables(c *C) { + var err error + var testkey1 = "testkey1" + var testkey2 = "testkey2" + var testkey3 = "testkey3" + bc := getBlueprintConfigForTest() + ctyMap := make(map[string]cty.Value) + err = bc.Config.ResolveGlobalVariables(ctyMap) + c.Assert(err, IsNil) + + // confirm plain string is unchanged and does not error + testCtyString := cty.StringVal("testval") + ctyMap[testkey1] = testCtyString + err = bc.Config.ResolveGlobalVariables(ctyMap) + c.Assert(err, IsNil) + c.Assert(ctyMap[testkey1], Equals, testCtyString) + + // confirm literal, non-global, variable is unchanged and does not error + testCtyString = cty.StringVal("((module.testval))") + ctyMap[testkey1] = testCtyString + err = bc.Config.ResolveGlobalVariables(ctyMap) + c.Assert(err, IsNil) + c.Assert(ctyMap[testkey1], Equals, testCtyString) + + // confirm failed resolution of a literal global + testCtyString = cty.StringVal("((var.test_global_var))") + ctyMap[testkey1] = testCtyString + err = bc.Config.ResolveGlobalVariables(ctyMap) + c.Assert(err, NotNil) + c.Assert(err.Error(), Matches, ".*Unsupported attribute;.*") + + // confirm successful resolution of literal globals in presence of other strings + testGlobalVarString := "test_global_string" + testGlobalValString := "testval" + testGlobalVarBool := "test_global_bool" + testGlobalValBool := "testval" + testPlainString := "plain-string" + bc.Config.Vars[testGlobalVarString] = testGlobalValString + bc.Config.Vars[testGlobalVarBool] = testGlobalValBool + testCtyString = cty.StringVal(fmt.Sprintf("((var.%s))", testGlobalVarString)) + testCtyBool := cty.StringVal(fmt.Sprintf("((var.%s))", testGlobalVarBool)) + ctyMap[testkey1] = testCtyString + ctyMap[testkey2] = testCtyBool + ctyMap[testkey3] = cty.StringVal(testPlainString) + err = bc.Config.ResolveGlobalVariables(ctyMap) + c.Assert(err, IsNil) + c.Assert(ctyMap[testkey1], Equals, cty.StringVal(testGlobalValString)) + c.Assert(ctyMap[testkey2], Equals, cty.StringVal(testGlobalValBool)) + c.Assert(ctyMap[testkey3], Equals, cty.StringVal(testPlainString)) +} diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 8c38e58823..764073d023 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -32,6 +32,12 @@ const ( roleLabel string = "ghpc_role" simpleVariableExp string = `^\$\((.*)\)$` anyVariableExp string = `\$\((.*)\)` + literalExp string = `^\(\((.*)\)\)$` + // the greediness and non-greediness of expression below is important + // consume all whitespace at beginning and end + // consume only up to first period to get variable source + // consume only up to whitespace to get variable name + literalSplitExp string = `^\(\([[:space:]]*(.*?)\.(.*?)[[:space:]]*\)\)$` ) // expand expands variables and strings in the yaml config. Used directly by @@ -42,6 +48,11 @@ func (bc *BlueprintConfig) expand() { log.Fatalf("failed to apply default backend to resource groups: %v", err) } + if err := bc.addDefaultValidators(); err != nil { + log.Fatalf( + "failed to update validators when expanding the config: %v", err) + } + if err := bc.combineLabels(); err != nil { log.Fatalf( "failed to update resources labels when expanding the config: %v", err) @@ -107,15 +118,6 @@ func getResourceVarName(resID string, varName string) string { return fmt.Sprintf("$(%s.%s)", resID, varName) } -func stringSliceContains(slice []string, value string) bool { - for _, elem := range slice { - if elem == value { - return true - } - } - return false -} - func getResourceInputMap(inputs []resreader.VarInfo) map[string]string { resInputs := make(map[string]string) for _, input := range inputs { @@ -412,7 +414,8 @@ func expandSimpleVariable( errorMessages["varNotFound"], varSource) } if refGrpIndex != context.groupIndex { - log.Fatalf("Unimplemented: references to other groups are not yet supported") + return "", fmt.Errorf("%s: resource %s was defined in group %d and called from group %d", + errorMessages["varInAnotherGroup"], varSource, refGrpIndex, context.groupIndex) } // Get the resource info @@ -553,6 +556,13 @@ func updateVariables( // expandVariables recurses through the data structures in the yaml config and // expands all variables func (bc *BlueprintConfig) expandVariables() { + for _, validator := range bc.Config.Validators { + err := updateVariables(varContext{yamlConfig: bc.Config}, validator.Inputs, make(map[string]int)) + if err != nil { + log.Fatalf("expandVariables: %v", err) + } + } + for iGrp, grp := range bc.Config.ResourceGroups { for iRes := range grp.Resources { context := varContext{ @@ -570,3 +580,62 @@ func (bc *BlueprintConfig) expandVariables() { } } } + +// this function adds default validators to the blueprint if none have been +// defined. default validators are only added for global variables that exist +func (bc *BlueprintConfig) addDefaultValidators() error { + if bc.Config.Validators != nil { + return nil + } + bc.Config.Validators = []validatorConfig{} + + _, projectIDExists := bc.Config.Vars["project_id"] + _, regionExists := bc.Config.Vars["region"] + _, zoneExists := bc.Config.Vars["zone"] + + if projectIDExists { + v := validatorConfig{ + Validator: testProjectExistsName.String(), + Inputs: map[string]interface{}{ + "project_id": "$(vars.project_id)", + }, + } + bc.Config.Validators = append(bc.Config.Validators, v) + } + + if projectIDExists && regionExists { + v := validatorConfig{ + Validator: testRegionExistsName.String(), + Inputs: map[string]interface{}{ + "project_id": "$(vars.project_id)", + "region": "$(vars.region)", + }, + } + bc.Config.Validators = append(bc.Config.Validators, v) + + } + + if projectIDExists && zoneExists { + v := validatorConfig{ + Validator: testZoneExistsName.String(), + Inputs: map[string]interface{}{ + "project_id": "$(vars.project_id)", + "zone": "$(vars.zone)", + }, + } + bc.Config.Validators = append(bc.Config.Validators, v) + } + + if projectIDExists && regionExists && zoneExists { + v := validatorConfig{ + Validator: testZoneInRegionName.String(), + Inputs: map[string]interface{}{ + "project_id": "$(vars.project_id)", + "region": "$(vars.region)", + "zone": "$(vars.zone)", + }, + } + bc.Config.Validators = append(bc.Config.Validators, v) + } + return nil +} diff --git a/pkg/config/validate.go b/pkg/config/validate.go index 99f49979e0..de85285e57 100644 --- a/pkg/config/validate.go +++ b/pkg/config/validate.go @@ -20,19 +20,31 @@ import ( "fmt" "log" "regexp" + "strings" "hpc-toolkit/pkg/resreader" "hpc-toolkit/pkg/sourcereader" + "hpc-toolkit/pkg/validators" "github.com/pkg/errors" "gopkg.in/yaml.v2" ) +const ( + validationErrorMsg = "validation failed due to the issues listed above" +) + // validate is the top-level function for running the validation suite. func (bc BlueprintConfig) validate() { if err := bc.validateVars(); err != nil { log.Fatal(err) } + + // variables should be validated before running validators + if err := bc.executeValidators(); err != nil { + log.Fatal(err) + } + if err := bc.validateResources(); err != nil { log.Fatal(err) } @@ -41,6 +53,60 @@ func (bc BlueprintConfig) validate() { } } +// performs validation of global variables +func (bc BlueprintConfig) executeValidators() error { + var errored, warned bool + implementedValidators := bc.getValidators() + + if bc.Config.ValidationLevel == validationIgnore { + return nil + } + + for _, validator := range bc.Config.Validators { + if f, ok := implementedValidators[validator.Validator]; ok { + err := f(validator) + if err != nil { + var prefix string + switch bc.Config.ValidationLevel { + case validationWarning: + warned = true + prefix = "warning: " + default: + errored = true + prefix = "error: " + } + log.Print(prefix, err) + log.Println() + } + } else { + errored = true + log.Printf("%s is not an implemented validator", validator.Validator) + } + } + + if warned || errored { + log.Println("validator failures can indicate a credentials problem.") + log.Println("troubleshooting info appears at:") + log.Println() + log.Println("https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/README.md#supplying-cloud-credentials-to-terraform") + log.Println() + log.Println("validation can be configured:") + log.Println("- treat failures as warnings by using the create command") + log.Println(" with the flag \"--validation-level WARNING\"") + log.Println("- can be disabled entirely by using the create command") + log.Println(" with the flag \"--validation-level IGNORE\"") + log.Println("- a custom set of validators can be configured following") + log.Println(" instructions at:") + log.Println() + log.Println("https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/README.md#blueprint-warnings-and-errors") + } + + if errored { + return fmt.Errorf(validationErrorMsg) + } + return nil +} + // validateVars checks the global variables for viable types func (bc BlueprintConfig) validateVars() error { vars := bc.Config.Vars @@ -170,3 +236,200 @@ func (bc BlueprintConfig) validateResourceSettings() error { } return nil } + +func (bc *BlueprintConfig) getValidators() map[string]func(validatorConfig) error { + allValidators := map[string]func(validatorConfig) error{ + testProjectExistsName.String(): bc.testProjectExists, + testRegionExistsName.String(): bc.testRegionExists, + testZoneExistsName.String(): bc.testZoneExists, + testZoneInRegionName.String(): bc.testZoneInRegion, + } + return allValidators +} + +// check that the keys in inputs and requiredInputs are identical sets of strings +func testInputList(function string, inputs map[string]interface{}, requiredInputs []string) error { + var errored bool + for _, requiredInput := range requiredInputs { + if _, found := inputs[requiredInput]; !found { + log.Printf("a required input %s was not provided to %s!", requiredInput, function) + errored = true + } + } + + if errored { + return fmt.Errorf("at least one required input was not provided to %s", function) + } + + // ensure that no extra inputs were provided by comparing length + if len(requiredInputs) != len(inputs) { + errStr := "only %v inputs %s should be provided to %s" + return fmt.Errorf(errStr, len(requiredInputs), requiredInputs, function) + } + + return nil +} + +func (bc *BlueprintConfig) testProjectExists(validator validatorConfig) error { + requiredInputs := []string{"project_id"} + funcName := testProjectExistsName.String() + funcErrorMsg := fmt.Sprintf("validator %s failed", funcName) + + if validator.Validator != funcName { + return fmt.Errorf("passed wrong validator to %s implementation", funcName) + } + + err := testInputList(validator.Validator, validator.Inputs, requiredInputs) + if err != nil { + log.Print(funcErrorMsg) + return err + } + + projectID, err := bc.getStringValue(validator.Inputs["project_id"]) + if err != nil { + log.Print(funcErrorMsg) + return err + } + + // err is nil or an error + err = validators.TestProjectExists(projectID) + if err != nil { + log.Print(funcErrorMsg) + } + return err +} + +func (bc *BlueprintConfig) testRegionExists(validator validatorConfig) error { + requiredInputs := []string{"project_id", "region"} + funcName := testRegionExistsName.String() + funcErrorMsg := fmt.Sprintf("validator %s failed", funcName) + + if validator.Validator != funcName { + return fmt.Errorf("passed wrong validator to %s implementation", funcName) + } + + err := testInputList(validator.Validator, validator.Inputs, requiredInputs) + if err != nil { + return err + } + + projectID, err := bc.getStringValue(validator.Inputs["project_id"]) + if err != nil { + log.Print(funcErrorMsg) + return err + } + region, err := bc.getStringValue(validator.Inputs["region"]) + if err != nil { + log.Print(funcErrorMsg) + return err + } + + // err is nil or an error + err = validators.TestRegionExists(projectID, region) + if err != nil { + log.Print(funcErrorMsg) + } + return err +} + +func (bc *BlueprintConfig) testZoneExists(validator validatorConfig) error { + requiredInputs := []string{"project_id", "zone"} + funcName := testZoneExistsName.String() + funcErrorMsg := fmt.Sprintf("validator %s failed", funcName) + + if validator.Validator != funcName { + return fmt.Errorf("passed wrong validator to %s implementation", funcName) + } + + err := testInputList(validator.Validator, validator.Inputs, requiredInputs) + if err != nil { + return err + } + + projectID, err := bc.getStringValue(validator.Inputs["project_id"]) + if err != nil { + log.Print(funcErrorMsg) + return err + } + zone, err := bc.getStringValue(validator.Inputs["zone"]) + if err != nil { + log.Print(funcErrorMsg) + return err + } + + // err is nil or an error + err = validators.TestZoneExists(projectID, zone) + if err != nil { + log.Print(funcErrorMsg) + } + return err +} + +func (bc *BlueprintConfig) testZoneInRegion(validator validatorConfig) error { + requiredInputs := []string{"project_id", "region", "zone"} + funcName := testZoneInRegionName.String() + funcErrorMsg := fmt.Sprintf("validator %s failed", funcName) + + if validator.Validator != funcName { + return fmt.Errorf("passed wrong validator to %s implementation", funcName) + } + + err := testInputList(validator.Validator, validator.Inputs, requiredInputs) + if err != nil { + return err + } + + projectID, err := bc.getStringValue(validator.Inputs["project_id"]) + if err != nil { + log.Print(funcErrorMsg) + return err + } + zone, err := bc.getStringValue(validator.Inputs["zone"]) + if err != nil { + log.Print(funcErrorMsg) + return err + } + region, err := bc.getStringValue(validator.Inputs["region"]) + if err != nil { + log.Print(funcErrorMsg) + return err + } + + // err is nil or an error + err = validators.TestZoneInRegion(projectID, zone, region) + if err != nil { + log.Print(funcErrorMsg) + } + return err +} + +// return the actual value of a global variable specified by the literal +// variable inputReference in form ((var.project_id)) +// if it is a literal global variable defined as a string, return value as string +// in all other cases, return empty string and error +func (bc *BlueprintConfig) getStringValue(inputReference interface{}) (string, error) { + varRef, ok := inputReference.(string) + if !ok { + return "", fmt.Errorf("the value %s cannot be cast to a string", inputReference) + } + + if IsLiteralVariable(varRef) { + varSlice := strings.Split(HandleLiteralVariable(varRef), ".") + varSrc := varSlice[0] + varName := varSlice[1] + + // because expand has already run, the global variable should have been + // checked for existence. handle if user has explicitly passed + // ((var.does_not_exit)) or ((not_a_varsrc.not_a_var)) + if varSrc == "var" { + if val, ok := bc.Config.Vars[varName]; ok { + valString, ok := val.(string) + if ok { + return valString, nil + } + return "", fmt.Errorf("the global variable %s is not a string", inputReference) + } + } + } + return "", fmt.Errorf("the value %s is not a global variable or was not defined", inputReference) +} diff --git a/pkg/config/validator_test.go b/pkg/config/validator_test.go index e61af949e0..7dd720fce0 100644 --- a/pkg/config/validator_test.go +++ b/pkg/config/validator_test.go @@ -27,6 +27,12 @@ import ( . "gopkg.in/check.v1" ) +const ( + missingRequiredInputRegex = "at least one required input was not provided to .*" + passedWrongValidatorRegex = "passed wrong validator to .*" + undefinedGlobalVariableRegex = ".* was not defined$" +) + func (s *MySuite) TestValidateResources(c *C) { bc := getBlueprintConfigForTest() bc.validateResources() @@ -160,3 +166,236 @@ func (s *MySuite) TestValidateOutputs(c *C) { expErr := fmt.Sprintf("%s.*", errorMessages["invalidOutput"]) c.Assert(err, ErrorMatches, expErr) } + +func (s *MySuite) TestAddDefaultValidators(c *C) { + bc := getBlueprintConfigForTest() + bc.addDefaultValidators() + c.Assert(bc.Config.Validators, HasLen, 0) + + bc.Config.Validators = nil + bc.Config.Vars["project_id"] = "not-a-project" + bc.addDefaultValidators() + c.Assert(bc.Config.Validators, HasLen, 1) + + bc.Config.Validators = nil + bc.Config.Vars["region"] = "us-central1" + bc.addDefaultValidators() + c.Assert(bc.Config.Validators, HasLen, 2) + + bc.Config.Validators = nil + bc.Config.Vars["zone"] = "us-central1-c" + bc.addDefaultValidators() + c.Assert(bc.Config.Validators, HasLen, 4) +} + +func (s *MySuite) TestTestInputList(c *C) { + var err error + var requiredInputs []string + + // SUCCESS: inputs is equal to required inputs without regard to ordering + requiredInputs = []string{"in0", "in1"} + inputs := map[string]interface{}{ + "in0": nil, + "in1": nil, + } + err = testInputList("testfunc", inputs, requiredInputs) + c.Assert(err, IsNil) + requiredInputs = []string{"in1", "in0"} + err = testInputList("testfunc", inputs, requiredInputs) + c.Assert(err, IsNil) + + // FAIL: inputs are a proper subset of required inputs + requiredInputs = []string{"in0", "in1", "in2"} + err = testInputList("testfunc", inputs, requiredInputs) + c.Assert(err, ErrorMatches, missingRequiredInputRegex) + + // FAIL: inputs intersect with required inputs but are not a proper subset + inputs = map[string]interface{}{ + "in0": nil, + "in1": nil, + "in3": nil, + } + err = testInputList("testfunc", inputs, requiredInputs) + c.Assert(err, ErrorMatches, missingRequiredInputRegex) + + // FAIL inputs are a proper superset of required inputs + inputs = map[string]interface{}{ + "in0": nil, + "in1": nil, + "in2": nil, + "in3": nil, + } + err = testInputList("testfunc", inputs, requiredInputs) + c.Assert(err, ErrorMatches, "only [0-9]+ inputs \\[.*\\] should be provided to testfunc") +} + +// return the actual value of a global variable specified by the literal +// variable inputReference in form ((var.project_id)) +// if it is a literal global variable defined as a string, return value as string +// in all other cases, return empty string and error +func (s *MySuite) TestGetStringValue(c *C) { + bc := getBlueprintConfigForTest() + bc.Config.Vars["goodvar"] = "testval" + bc.Config.Vars["badvar"] = 2 + + // test non-string values return error + _, err := bc.getStringValue(2) + c.Assert(err, Not(IsNil)) + + // test strings that are not literal variables return error and empty string + strVal, err := bc.getStringValue("hello") + c.Assert(err, Not(IsNil)) + c.Assert(strVal, Equals, "") + + // test literal variables that refer to strings return their value + strVal, err = bc.getStringValue("(( var.goodvar ))") + c.Assert(err, IsNil) + c.Assert(strVal, Equals, bc.Config.Vars["goodvar"]) + + // test literal variables that refer to non-strings return error + _, err = bc.getStringValue("(( var.badvar ))") + c.Assert(err, Not(IsNil)) +} + +func (s *MySuite) TestExecuteValidators(c *C) { + bc := getBlueprintConfigForTest() + bc.Config.Validators = []validatorConfig{ + { + Validator: "unimplemented-validator", + Inputs: map[string]interface{}{}, + }, + } + + err := bc.executeValidators() + c.Assert(err, ErrorMatches, validationErrorMsg) + + bc.Config.Validators = []validatorConfig{ + { + Validator: testProjectExistsName.String(), + Inputs: map[string]interface{}{}, + }, + } + + err = bc.executeValidators() + c.Assert(err, ErrorMatches, validationErrorMsg) +} + +// this function tests that the "gateway" functions in this package for our +// validators fail under various conditions; it does not test the actual Cloud +// API calls in the validators package; we will defer success testing until the +// development of mock functions for Cloud API calls +func (s *MySuite) TestProjectExistsValidator(c *C) { + var err error + bc := getBlueprintConfigForTest() + emptyValidator := validatorConfig{} + + // test validator fails for config without validator id + err = bc.testProjectExists(emptyValidator) + c.Assert(err, ErrorMatches, passedWrongValidatorRegex) + + // test validator fails for config without any inputs + projectValidator := validatorConfig{ + Validator: testProjectExistsName.String(), + Inputs: map[string]interface{}{}, + } + err = bc.testProjectExists(projectValidator) + c.Assert(err, ErrorMatches, missingRequiredInputRegex) + + // test validators fail when input global variables are undefined + projectValidator.Inputs["project_id"] = "((var.project_id))" + err = bc.testProjectExists(projectValidator) + c.Assert(err, ErrorMatches, undefinedGlobalVariableRegex) + + // TODO: implement a mock client to test success of test_project_exists +} + +func (s *MySuite) TestRegionExistsValidator(c *C) { + var err error + bc := getBlueprintConfigForTest() + emptyValidator := validatorConfig{} + + // test validator fails for config without validator id + err = bc.testRegionExists(emptyValidator) + c.Assert(err, ErrorMatches, passedWrongValidatorRegex) + + // test validator fails for config without any inputs + regionValidator := validatorConfig{ + Validator: testRegionExistsName.String(), + Inputs: map[string]interface{}{}, + } + err = bc.testRegionExists(regionValidator) + c.Assert(err, ErrorMatches, missingRequiredInputRegex) + + // test validators fail when input global variables are undefined + regionValidator.Inputs["project_id"] = "((var.project_id))" + regionValidator.Inputs["region"] = "((var.region))" + err = bc.testRegionExists(regionValidator) + c.Assert(err, ErrorMatches, undefinedGlobalVariableRegex) + bc.Config.Vars["project_id"] = "invalid-project" + err = bc.testRegionExists(regionValidator) + c.Assert(err, ErrorMatches, undefinedGlobalVariableRegex) + + // TODO: implement a mock client to test success of test_region_exists +} + +func (s *MySuite) TestZoneExistsValidator(c *C) { + var err error + bc := getBlueprintConfigForTest() + emptyValidator := validatorConfig{} + + // test validator fails for config without validator id + err = bc.testZoneExists(emptyValidator) + c.Assert(err, ErrorMatches, passedWrongValidatorRegex) + + // test validator fails for config without any inputs + zoneValidator := validatorConfig{ + Validator: testZoneExistsName.String(), + Inputs: map[string]interface{}{}, + } + err = bc.testZoneExists(zoneValidator) + c.Assert(err, ErrorMatches, missingRequiredInputRegex) + + // test validators fail when input global variables are undefined + zoneValidator.Inputs["project_id"] = "((var.project_id))" + zoneValidator.Inputs["zone"] = "((var.zone))" + err = bc.testZoneExists(zoneValidator) + c.Assert(err, ErrorMatches, undefinedGlobalVariableRegex) + bc.Config.Vars["project_id"] = "invalid-project" + err = bc.testZoneExists(zoneValidator) + c.Assert(err, ErrorMatches, undefinedGlobalVariableRegex) + + // TODO: implement a mock client to test success of test_zone_exists +} + +func (s *MySuite) TestZoneInRegionValidator(c *C) { + var err error + bc := getBlueprintConfigForTest() + emptyValidator := validatorConfig{} + + // test validator fails for config without validator id + err = bc.testZoneInRegion(emptyValidator) + c.Assert(err, ErrorMatches, passedWrongValidatorRegex) + + // test validator fails for config without any inputs + zoneInRegionValidator := validatorConfig{ + Validator: testZoneInRegionName.String(), + Inputs: map[string]interface{}{}, + } + err = bc.testZoneInRegion(zoneInRegionValidator) + c.Assert(err, ErrorMatches, missingRequiredInputRegex) + + // test validators fail when input global variables are undefined + zoneInRegionValidator.Inputs["project_id"] = "((var.project_id))" + zoneInRegionValidator.Inputs["region"] = "((var.region))" + zoneInRegionValidator.Inputs["zone"] = "((var.zone))" + err = bc.testZoneInRegion(zoneInRegionValidator) + c.Assert(err, ErrorMatches, undefinedGlobalVariableRegex) + bc.Config.Vars["project_id"] = "invalid-project" + err = bc.testZoneInRegion(zoneInRegionValidator) + c.Assert(err, ErrorMatches, undefinedGlobalVariableRegex) + bc.Config.Vars["zone"] = "invalid-zone" + err = bc.testZoneInRegion(zoneInRegionValidator) + c.Assert(err, ErrorMatches, undefinedGlobalVariableRegex) + + // TODO: implement a mock client to test success of test_zone_in_region +} diff --git a/pkg/reswriter/hcl_utils.go b/pkg/reswriter/hcl_utils.go index 4540063607..80f893494a 100644 --- a/pkg/reswriter/hcl_utils.go +++ b/pkg/reswriter/hcl_utils.go @@ -16,154 +16,31 @@ package reswriter import ( "fmt" - "log" - "regexp" - "strings" + "path/filepath" - "gopkg.in/yaml.v2" - - "hpc-toolkit/pkg/config" + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/zclconf/go-cty/cty" ) -// interfaceStruct is a struct wrapper for converting interface data structures -// to yaml flow style: one line wrapped in {} for maps and [] for lists. -type interfaceStruct struct { - Elem interface{} `yaml:",flow"` -} - -func getType(obj interface{}) string { - // This does not handle variables with arbitrary types - str, ok := obj.(string) - if !ok { // We received a nil value. - return "null" - } - if strings.HasPrefix(str, "{") { - return "map" - } - if strings.HasPrefix(str, "[") { - return "list" - } - return "string" -} - -func isLiteralVariable(str string) bool { - match, err := regexp.MatchString(beginLiteralExp, str) - if err != nil { - log.Fatalf("Failed checking if variable is a literal: %v", err) - } - return match -} - -func handleLiteralVariable(str string) string { - re := regexp.MustCompile(fullLiteralExp) - contents := re.FindStringSubmatch(str) - if len(contents) != 2 { - log.Fatalf("Incorrectly formatted literal variable: %s", str) - } - - return contents[1] -} - -func handleData(val interface{}) interface{} { - str, ok := val.(string) - if !ok { - // We only need to act on strings - return val +func writeHclAttributes(vars map[string]cty.Value, dst string) error { + if err := createBaseFile(dst); err != nil { + return fmt.Errorf("error creating variables file %v: %v", filepath.Base(dst), err) } - if isLiteralVariable(str) { - return handleLiteralVariable(str) - } else if !strings.HasPrefix(str, "[") && - !strings.HasPrefix(str, "{") { - return fmt.Sprintf("\"%s\"", str) - } - return str -} - -func updateStringsInInterface(value interface{}) (interface{}, error) { - var err error - switch typedValue := value.(type) { - case []interface{}: - for i := 0; i < len(typedValue); i++ { - typedValue[i], err = updateStringsInInterface(typedValue[i]) - if err != nil { - break - } - } - return typedValue, err - case map[string]interface{}: - retMap := map[string]interface{}{} - for k, v := range typedValue { - retMap[handleData(k).(string)], err = updateStringsInInterface(v) - if err != nil { - break - } - } - return retMap, err - default: - return handleData(value), err - } -} -func updateStringsInMap(interfaceMap map[string]interface{}) error { - var err error - for key, value := range interfaceMap { - interfaceMap[key], err = updateStringsInInterface(value) - if err != nil { - break - } - } - return err -} + // Create hcl body + hclFile := hclwrite.NewEmptyFile() + hclBody := hclFile.Body() -func updateStringsInConfig(yamlConfig *config.YamlConfig, kind string) { - for iGrp, grp := range yamlConfig.ResourceGroups { - for iRes := 0; iRes < len(grp.Resources); iRes++ { - if grp.Resources[iRes].Kind != kind { - continue - } - err := updateStringsInMap( - yamlConfig.ResourceGroups[iGrp].Resources[iRes].Settings) - if err != nil { - log.Fatalf("updateStringsInConfig: %v", err) - } - } + // for each variable + for k, v := range vars { + // Write attribute + hclBody.SetAttributeValue(k, v) } -} -func convertToYaml(wrappedInterface *interfaceStruct) (string, error) { - by, err := yaml.Marshal(wrappedInterface) + // Write file + err := appendHCLToFile(dst, hclFile.Bytes()) if err != nil { - return "", err - } - return strings.TrimSuffix( - strings.ReplaceAll(string(by[6:]), "'", ""), "\n"), err -} - -func flattenInterfaceMap( - interfaceMap map[string]interface{}, wrapper *interfaceStruct) error { - for k, v := range interfaceMap { - wrapper.Elem = v - yamlStr, err := convertToYaml(wrapper) - if err != nil { - return err - } - interfaceMap[k] = yamlStr - } - return nil -} - -func flattenToHCLStrings(yamlConfig *config.YamlConfig, kind string) { - wrapper := interfaceStruct{Elem: nil} - for iGrp, grp := range yamlConfig.ResourceGroups { - for iRes := 0; iRes < len(grp.Resources); iRes++ { - if grp.Resources[iRes].Kind != kind { - continue - } - err := flattenInterfaceMap( - yamlConfig.ResourceGroups[iGrp].Resources[iRes].Settings, &wrapper) - if err != nil { - log.Fatalf("flattenToHCLStrings: %v", err) - } - } + return fmt.Errorf("error writing HCL to %v: %v", filepath.Base(dst), err) } + return err } diff --git a/pkg/reswriter/license.go b/pkg/reswriter/license.go index ecbcbca9c4..80dfde15f1 100644 --- a/pkg/reswriter/license.go +++ b/pkg/reswriter/license.go @@ -17,7 +17,7 @@ package reswriter const license string = `/** - * Copyright 2021 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/reswriter/packerwriter.go b/pkg/reswriter/packerwriter.go index 1b68c6b956..c8714cf4c8 100644 --- a/pkg/reswriter/packerwriter.go +++ b/pkg/reswriter/packerwriter.go @@ -18,14 +18,14 @@ package reswriter import ( "fmt" - "os" "path/filepath" - "text/template" "hpc-toolkit/pkg/config" + + "github.com/zclconf/go-cty/cty" ) -const packerAutoVarFilename = "variables.auto.pkrvars.hcl" +const packerAutoVarFilename = "defaults.auto.pkrvars.hcl" // PackerWriter writes packer to the blueprint folder type PackerWriter struct { @@ -40,13 +40,6 @@ func (w *PackerWriter) addNumResources(value int) { w.numResources += value } -// prepareToWrite makes any resource kind specific changes to the config before -// writing to the blueprint directory -func (w PackerWriter) prepareToWrite(yamlConfig *config.YamlConfig) { - updateStringsInConfig(yamlConfig, "packer") - flattenToHCLStrings(yamlConfig, "packer") -} - func printPackerInstructions(grpPath string) { printInstructionsPreamble("Packer", grpPath) fmt.Printf(" cd %s\n", grpPath) @@ -61,8 +54,19 @@ func (w PackerWriter) writeResourceLevel(yamlConfig *config.YamlConfig, bpDirect if res.Kind != "packer" { continue } + + ctySettings, err := config.ConvertMapToCty(res.Settings) + + if err != nil { + return fmt.Errorf( + "error converting global vars to cty for writing: %v", err) + } + err = yamlConfig.ResolveGlobalVariables(ctySettings) + if err != nil { + return err + } resPath := filepath.Join(groupPath, res.ID) - err := writePackerAutoVariables(packerAutoVarFilename, res, resPath) + err = writePackerAutovars(ctySettings, resPath) if err != nil { return err } @@ -72,42 +76,14 @@ func (w PackerWriter) writeResourceLevel(yamlConfig *config.YamlConfig, bpDirect return nil } -func writePackerAutoVariables( - tmplFilename string, resource config.Resource, destPath string) error { - tmplText := getTemplate(fmt.Sprintf("%s.tmpl", tmplFilename)) - - funcMap := template.FuncMap{ - "getType": getType, - } - tmpl, err := template.New(tmplFilename).Funcs(funcMap).Parse(tmplText) - - if err != nil { - return fmt.Errorf( - "failed to create template %s when writing packer resource at %s: %v", - tmplFilename, resource.Source, err) - } - if tmpl == nil { - return fmt.Errorf( - "failed to parse the %s template", tmplFilename) - } - - outputPath := filepath.Join(destPath, tmplFilename) - outputFile, err := os.Create(outputPath) - if err != nil { - return fmt.Errorf( - "failed to create packer file %s: %v", tmplFilename, err) - } - if err := tmpl.Execute(outputFile, resource); err != nil { - return fmt.Errorf( - "failed to write template for %s file when writing packer resource %s: %e", - tmplFilename, resource.ID, err) - } - return nil +func writePackerAutovars(vars map[string]cty.Value, dst string) error { + packerAutovarsPath := filepath.Join(dst, packerAutoVarFilename) + err := writeHclAttributes(vars, packerAutovarsPath) + return err } // writeResourceGroups writes any needed files to the top and resource levels // of the blueprint func (w PackerWriter) writeResourceGroups(yamlConfig *config.YamlConfig, bpDirectory string) error { - w.prepareToWrite(yamlConfig) return w.writeResourceLevel(yamlConfig, bpDirectory) } diff --git a/pkg/reswriter/reswriter.go b/pkg/reswriter/reswriter.go index 776319d5b9..baa933d986 100644 --- a/pkg/reswriter/reswriter.go +++ b/pkg/reswriter/reswriter.go @@ -18,7 +18,6 @@ package reswriter import ( - "embed" "fmt" "hpc-toolkit/pkg/blueprintio" "hpc-toolkit/pkg/config" @@ -28,11 +27,6 @@ import ( "path/filepath" ) -const ( - beginLiteralExp string = `^\(\(.*$` - fullLiteralExp string = `^\(\((.*)\)\)$` -) - // ResWriter interface for writing resources to a blueprint type ResWriter interface { getNumResources() int @@ -45,9 +39,6 @@ var kinds = map[string]ResWriter{ "packer": new(PackerWriter), } -//go:embed *.tmpl -var templatesFS embed.FS - func factory(kind string) ResWriter { writer, exists := kinds[kind] if !exists { @@ -58,18 +49,13 @@ func factory(kind string) ResWriter { return writer } -func getTemplate(filename string) string { - // Create path to template from the embedded template FS - tmplText, err := templatesFS.ReadFile(filename) - if err != nil { - log.Fatalf("reswriter: %v", err) - } - return string(tmplText) -} - func copySource(blueprintPath string, resourceGroups *[]config.ResourceGroup) { for iGrp, grp := range *resourceGroups { for iRes, resource := range grp.Resources { + if sourcereader.IsGitHubPath(resource.Source) { + continue + } + /* Copy source files */ resourceName := filepath.Base(resource.Source) (*resourceGroups)[iGrp].Resources[iRes].ResourceName = resourceName @@ -104,11 +90,11 @@ func printInstructionsPreamble(kind string, path string) { } // WriteBlueprint writes the blueprint using resources defined in config. -func WriteBlueprint(yamlConfig *config.YamlConfig, bpDirectory string) { +func WriteBlueprint(yamlConfig *config.YamlConfig, bpDirectory string) error { blueprintio := blueprintio.GetBlueprintIOLocal() bpDirectoryPath := filepath.Join(bpDirectory, yamlConfig.BlueprintName) if err := blueprintio.CreateDirectory(bpDirectoryPath); err != nil { - log.Fatalf("failed to create a directory for blueprints: %v", err) + return fmt.Errorf("failed to create a directory for blueprints: %w", err) } copySource(bpDirectoryPath, &yamlConfig.ResourceGroups) @@ -116,8 +102,9 @@ func WriteBlueprint(yamlConfig *config.YamlConfig, bpDirectory string) { if writer.getNumResources() > 0 { err := writer.writeResourceGroups(yamlConfig, bpDirectory) if err != nil { - log.Fatalf("error writing resources to blueprint: %v", err) + return fmt.Errorf("error writing resources to blueprint: %w", err) } } } + return nil } diff --git a/pkg/reswriter/reswriter_test.go b/pkg/reswriter/reswriter_test.go index ae61e64389..0b4558102a 100644 --- a/pkg/reswriter/reswriter_test.go +++ b/pkg/reswriter/reswriter_test.go @@ -89,7 +89,7 @@ func getYamlConfigForTest() config.YamlConfig { }, } testResourceGroups := []config.ResourceGroup{ - config.ResourceGroup{ + { Resources: []config.Resource{testResource, testResourceWithLabels}, }, } @@ -102,14 +102,6 @@ func getYamlConfigForTest() config.YamlConfig { return testYamlConfig } -func createTestApplyFunctions(config config.YamlConfig) [][]map[string]string { - applyFuncs := make([][]map[string]string, len(config.ResourceGroups)) - for iGrp, group := range config.ResourceGroups { - applyFuncs[iGrp] = make([]map[string]string, len(group.Resources)) - } - return applyFuncs -} - // Tests // reswriter.go @@ -117,157 +109,11 @@ func (s *MySuite) TestWriteBlueprint(c *C) { testYamlConfig := getYamlConfigForTest() blueprintName := "blueprints_TestWriteBlueprint" testYamlConfig.BlueprintName = blueprintName - WriteBlueprint(&testYamlConfig, testDir) -} - -func (s *MySuite) TestFlattenInterfaceMap(c *C) { - wrapper := interfaceStruct{Elem: nil} - inputMaps := []interface{}{ - // Just a string - "str1", - // map of strings - map[string]interface{}{ - "str1": "val1", - "str2": "val2", - }, - // slice of strings - []interface{}{"str1", "str2"}, - // map of maps - map[string]interface{}{ - "map1": map[string]interface{}{}, - "map2": map[string]interface{}{ - "str1": "val1", - "str2": "val2", - }, - }, - // slice of slices - []interface{}{ - []interface{}{}, - []interface{}{"str1", "str2"}, - }, - // map of slice of map - map[string]interface{}{ - "slice": []map[string]interface{}{ - map[string]interface{}{ - "str1": "val1", - "str2": "val2", - }, - }, - }, - // empty map - map[string]interface{}{}, - // empty slice - []interface{}{}, - } - // map of all 3 - inputMapAllThree := map[string]interface{}{ - "str": "val", - "map": map[string]interface{}{ - "str1": "val1", - "str2": "val2", - }, - "slice": []interface{}{"str1", "str2"}, - } - stringMapContents := "{str1: val1, str2: val2}" - stringSliceContents := "[str1, str2]" - expectedOutputs := []string{ - "str1", // Just a string - stringMapContents, // map of strings - stringSliceContents, // slice of strings - fmt.Sprintf("{map1: {}, map2: %s}", stringMapContents), // map of maps - fmt.Sprintf("[[], %s]", stringSliceContents), // slice of slices - fmt.Sprintf("{slice: [%s]}", stringMapContents), // map of slice of map - "{}", - "[]", - } - - // Test the test setup - c.Assert(len(inputMaps), Equals, len(expectedOutputs)) - - // Test common cases - mapWrapper := make(map[string]interface{}) - for i := range inputMaps { - mapWrapper["key"] = inputMaps[i] - err := flattenInterfaceMap(mapWrapper, &wrapper) - c.Assert(err, IsNil) - c.Assert(mapWrapper["key"], Equals, expectedOutputs[i]) - } - - // Test complicated case - mapWrapper["key"] = inputMapAllThree - err := flattenInterfaceMap(mapWrapper, &wrapper) - c.Assert(err, IsNil) - c.Assert( - strings.Contains(mapWrapper["key"].(string), "str: val"), Equals, true) - mapString := fmt.Sprintf("map: %s", stringMapContents) - c.Assert( - strings.Contains(mapWrapper["key"].(string), mapString), Equals, true) - sliceString := fmt.Sprintf("slice: %s", stringSliceContents) - c.Assert( - strings.Contains(mapWrapper["key"].(string), sliceString), Equals, true) -} - -func testHandlePrimitivesCreateMap() map[string]interface{} { - // String test variables - addQuotes := "addQuotes" - noQuotes := "((noQuotes))" - - // Composite test variables - testMap := map[string]interface{}{ - "stringMap": addQuotes, - "variableMap": noQuotes, - "deep": map[string]interface{}{ - "slice": []interface{}{addQuotes, noQuotes}, - }, - } - testSlice := []interface{}{addQuotes, noQuotes} - - return map[string]interface{}{ - "string": addQuotes, - "variable": noQuotes, - "map": testMap, - "slice": testSlice, - } -} - -func testHandlePrimitivesHelper(c *C, varMap map[string]interface{}) { - addQuotesExpected := fmt.Sprintf("\"%s\"", "addQuotes") - noQuotesExpected := "noQuotes" - - // Test top level - c.Assert(varMap["string"], Equals, addQuotesExpected) - c.Assert(varMap["variable"], Equals, noQuotesExpected) - - // Test map - interfaceMap := varMap["map"].(map[string]interface{}) - c.Assert(interfaceMap["\"stringMap\""], - Equals, - addQuotesExpected) - c.Assert(interfaceMap["\"variableMap\""], Equals, noQuotesExpected) - interfaceMap = interfaceMap["\"deep\""].(map[string]interface{}) - interfaceSlice := interfaceMap["\"slice\""].([]interface{}) - c.Assert(interfaceSlice[0], Equals, addQuotesExpected) - c.Assert(interfaceSlice[1], Equals, noQuotesExpected) - - // Test slice - interfaceSlice = varMap["slice"].([]interface{}) - c.Assert(interfaceSlice[0], Equals, addQuotesExpected) - c.Assert(interfaceSlice[1], Equals, noQuotesExpected) -} - -func (s *MySuite) TestUpdateStrings(c *C) { - yamlConfig := getYamlConfigForTest() - - // Setup Vars - yamlConfig.Vars = testHandlePrimitivesCreateMap() - yamlConfig.ResourceGroups[0].Resources[0].Settings = - testHandlePrimitivesCreateMap() - - updateStringsInConfig(&yamlConfig, "terraform") - - testHandlePrimitivesHelper( - c, yamlConfig.ResourceGroups[0].Resources[0].Settings) - + err := WriteBlueprint(&testYamlConfig, testDir) + c.Check(err, IsNil) + // Overwriting the blueprint fails + err = WriteBlueprint(&testYamlConfig, testDir) + c.Check(err, NotNil) } // tfwriter.go @@ -367,7 +213,7 @@ func (s *MySuite) TestCreateBaseFile(c *C) { c.Assert(fi.Name(), Equals, baseFilename) c.Assert(fi.Size() > 0, Equals, true) c.Assert(fi.IsDir(), Equals, false) - b, err := ioutil.ReadFile(goodPath) + b, _ := ioutil.ReadFile(goodPath) c.Assert(strings.Contains(string(b), "Licensed under the Apache License"), Equals, true) @@ -400,39 +246,6 @@ func stringExistsInFile(str string, filename string) (bool, error) { return strings.Contains(string(b), str), nil } -// hcl_utils.go -func (s *MySuite) TestFlattenToHCLStrings(c *C) { - testConfig := getYamlConfigForTest() - flattenToHCLStrings(&testConfig, "terraform") -} - -func (s *MySuite) TestGetType(c *C) { - - // string - testString := "test string" - ret := getType(testString) - c.Assert(ret, Equals, "string") - - // map - testMap := "{testMap: testVal}" - ret = getType(testMap) - c.Assert(ret, Equals, "map") - - // list - testList := "[testList0,testList]" - ret = getType(testList) - c.Assert(ret, Equals, "list") - - // non-string input - testNull := 42 // random int - ret = getType(testNull) - c.Assert(ret, Equals, "null") - - // nil input - ret = getType(nil) - c.Assert(ret, Equals, "null") -} - func (s *MySuite) TestWriteMain(c *C) { // Setup testMainDir := filepath.Join(testDir, "TestWriteMain") @@ -490,7 +303,7 @@ func (s *MySuite) TestWriteMain(c *C) { testResourceWithWrap := config.Resource{ ID: "test_resource_with_wrap", WrapSettingsWith: map[string][]string{ - "wrappedSetting": []string{"list(flatten(", "))"}, + "wrappedSetting": {"list(flatten(", "))"}, }, Settings: map[string]interface{}{ "wrappedSetting": []interface{}{"val1", "val2"}, @@ -653,15 +466,24 @@ func (s *MySuite) TestWriteResourceLevel_PackerWriter(c *C) { c.Assert(err, IsNil) } -func (s *MySuite) TestWritePackerAutoVariables(c *C) { - // The happy path is tested outside of this funcation already +func (s *MySuite) TestWritePackerAutoVars(c *C) { + testYamlConfig := getYamlConfigForTest() + testYamlConfig.Vars["testkey"] = "testval" + ctyVars, _ := config.ConvertMapToCty(testYamlConfig.Vars) - // Bad tmplFilename + // fail writing to a bad path badDestPath := "not/a/real/path" - err := writePackerAutoVariables( - packerAutoVarFilename, config.Resource{}, badDestPath) - expErr := "failed to create packer file .*" + err := writePackerAutovars(ctyVars, badDestPath) + expErr := fmt.Sprintf("error creating variables file %s:.*", packerAutoVarFilename) c.Assert(err, ErrorMatches, expErr) + + testPackerTemplateDir := filepath.Join(testDir, "TestWritePackerTemplate") + if err := os.Mkdir(testPackerTemplateDir, 0755); err != nil { + log.Fatalf("Failed to create test dir for creating %s file", packerAutoVarFilename) + } + err = writePackerAutovars(ctyVars, testPackerTemplateDir) + c.Assert(err, IsNil) + } func TestMain(m *testing.M) { diff --git a/pkg/reswriter/tfwriter.go b/pkg/reswriter/tfwriter.go index 9927b5d83f..ee1b65f613 100644 --- a/pkg/reswriter/tfwriter.go +++ b/pkg/reswriter/tfwriter.go @@ -17,7 +17,6 @@ package reswriter import ( - "encoding/json" "fmt" "os" "path/filepath" @@ -27,9 +26,9 @@ import ( "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/hcl/v2/hclwrite" "github.com/zclconf/go-cty/cty" - ctyJson "github.com/zclconf/go-cty/cty/json" "hpc-toolkit/pkg/config" + "hpc-toolkit/pkg/sourcereader" ) // TFWriter writes terraform to the blueprint folder @@ -51,10 +50,10 @@ func (w *TFWriter) addNumResources(value int) { // license and any other boilerplate func createBaseFile(path string) error { baseFile, err := os.Create(path) - defer baseFile.Close() if err != nil { return err } + defer baseFile.Close() _, err = baseFile.WriteString(license) return err } @@ -76,31 +75,6 @@ func appendHCLToFile(path string, hclBytes []byte) error { return nil } -func convertToCty(val interface{}) (cty.Value, error) { - // Convert to JSON bytes - jsonBytes, err := json.Marshal(val) - if err != nil { - return cty.Value{}, err - } - - // Unmarshal JSON into cty - simpleJSON := ctyJson.SimpleJSONValue{} - simpleJSON.UnmarshalJSON(jsonBytes) - return simpleJSON.Value, nil -} - -func convertMapToCty(iMap map[string]interface{}) (map[string]cty.Value, error) { - cMap := make(map[string]cty.Value) - for k, v := range iMap { - convertedVal, err := convertToCty(v) - if err != nil { - return cMap, err - } - cMap[k] = convertedVal - } - return cMap, nil -} - func writeOutputs( resources []config.Resource, dst string, @@ -144,25 +118,7 @@ func writeOutputs( func writeTfvars(vars map[string]cty.Value, dst string) error { // Create file tfvarsPath := filepath.Join(dst, "terraform.tfvars") - if err := createBaseFile(tfvarsPath); err != nil { - return fmt.Errorf("error creating terraform.tfvars file: %v", err) - } - - // Create hcl body - hclFile := hclwrite.NewEmptyFile() - hclBody := hclFile.Body() - - // for each variable - for k, v := range vars { - // Write attribute - hclBody.SetAttributeValue(k, v) - } - - // Write file - err := appendHCLToFile(tfvarsPath, hclFile.Bytes()) - if err != nil { - return fmt.Errorf("error writing HCL to terraform.tfvars file: %v", err) - } + err := writeHclAttributes(vars, tfvarsPath) return err } @@ -245,7 +201,7 @@ func writeMain( // Write Terraform backend if needed if tfBackend.Type != "" { - tfConfig, err := convertMapToCty(tfBackend.Configuration) + tfConfig, err := config.ConvertMapToCty(tfBackend.Configuration) if err != nil { errString := "error converting terraform backend configuration to cty when writing main.tf: %v" return fmt.Errorf(errString, err) @@ -262,7 +218,7 @@ func writeMain( // For each resource: for _, res := range resources { // Convert settings to cty.Value - ctySettings, err := convertMapToCty(res.Settings) + ctySettings, err := config.ConvertMapToCty(res.Settings) if err != nil { return fmt.Errorf( "error converting setting in resource %s to cty when writing main.tf: %v", @@ -274,7 +230,13 @@ func writeMain( moduleBody := moduleBlock.Body() // Add source attribute - moduleSource := cty.StringVal(fmt.Sprintf("./modules/%s", res.ResourceName)) + var moduleSource cty.Value + if sourcereader.IsGitHubPath(res.Source) { + moduleSource = cty.StringVal(res.Source) + } else { + moduleSource = cty.StringVal(fmt.Sprintf("./modules/%s", res.ResourceName)) + } + moduleBody.SetAttributeValue("source", moduleSource) // For each Setting @@ -395,9 +357,8 @@ func writeVersions(dst string) error { func printTerraformInstructions(grpPath string) { printInstructionsPreamble("Terraform", grpPath) - fmt.Printf(" cd %s\n", grpPath) - fmt.Println(" terraform init") - fmt.Println(" terraform apply") + fmt.Printf(" terraform -chdir=%s init\n", grpPath) + fmt.Printf(" terraform -chdir=%s apply\n", grpPath) } // writeTopLevel writes any needed files to the top layer of the blueprint @@ -406,7 +367,7 @@ func (w TFWriter) writeResourceGroups( bpDirectory string, ) error { bpName := yamlConfig.BlueprintName - ctyVars, err := convertMapToCty(yamlConfig.Vars) + ctyVars, err := config.ConvertMapToCty(yamlConfig.Vars) if err != nil { return fmt.Errorf( "error converting global vars to cty for writing: %v", err) diff --git a/pkg/reswriter/variables.auto.pkrvars.hcl.tmpl b/pkg/reswriter/variables.auto.pkrvars.hcl.tmpl deleted file mode 100644 index 78206e7374..0000000000 --- a/pkg/reswriter/variables.auto.pkrvars.hcl.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -/** -* Copyright 2021 Google LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. - */ - -{{ range $setting, $value := .Settings -}} -{{$setting}} = {{$value}} -{{ end -}} diff --git a/pkg/sourcereader/github.go b/pkg/sourcereader/github.go index 2f42755dc0..9689befd4a 100644 --- a/pkg/sourcereader/github.go +++ b/pkg/sourcereader/github.go @@ -20,6 +20,7 @@ import ( "hpc-toolkit/pkg/resreader" "io/ioutil" "os" + "path/filepath" "github.com/hashicorp/go-getter" ) @@ -63,17 +64,18 @@ func (r GitHubSourceReader) GetResourceInfo(resPath string, kind string) (resrea resDir, err := ioutil.TempDir("", "git-module-*") defer os.RemoveAll(resDir) + writeDir := filepath.Join(resDir, "mod") if err != nil { return resreader.ResourceInfo{}, err } - if err := copyGitHubResources(resPath, resDir); err != nil { + if err := copyGitHubResources(resPath, writeDir); err != nil { return resreader.ResourceInfo{}, fmt.Errorf("failed to clone GitHub resource at %s to tmp dir %s: %v", - resPath, resDir, err) + resPath, writeDir, err) } reader := resreader.Factory(kind) - return reader.GetInfo(resDir) + return reader.GetInfo(writeDir) } // GetResource copies the GitHub source to a provided destination (the blueprint directory) @@ -84,14 +86,15 @@ func (r GitHubSourceReader) GetResource(resPath string, copyPath string) error { resDir, err := ioutil.TempDir("", "git-module-*") defer os.RemoveAll(resDir) + writeDir := filepath.Join(resDir, "mod") if err != nil { return err } - if err := copyGitHubResources(resPath, resDir); err != nil { + if err := copyGitHubResources(resPath, writeDir); err != nil { return fmt.Errorf("failed to clone GitHub resource at %s to tmp dir %s: %v", - resPath, resDir, err) + resPath, writeDir, err) } - return copyFromPath(resDir, copyPath) + return copyFromPath(writeDir, copyPath) } diff --git a/pkg/sourcereader/github_test.go b/pkg/sourcereader/github_test.go index 48e3eeb265..d40ed57733 100644 --- a/pkg/sourcereader/github_test.go +++ b/pkg/sourcereader/github_test.go @@ -38,6 +38,16 @@ func (s *MySuite) TestCopyGitHubResources(c *C) { c.Assert(fInfo.Name(), Equals, "terraform_validate") c.Assert(fInfo.Size() > 0, Equals, true) c.Assert(fInfo.IsDir(), Equals, false) + + // Success via HTTPS (Root directory) + destDirForHTTPSRootDir := filepath.Join(destDir, "https-rootdir") + err = copyGitHubResources("github.com/terraform-google-modules/terraform-google-service-accounts.git?ref=v4.1.1", destDirForHTTPSRootDir) + c.Assert(err, IsNil) + fInfo, err = os.Stat(filepath.Join(destDirForHTTPSRootDir, "main.tf")) + c.Assert(err, IsNil) + c.Assert(fInfo.Name(), Equals, "main.tf") + c.Assert(fInfo.Size() > 0, Equals, true) + c.Assert(fInfo.IsDir(), Equals, false) } func (s *MySuite) TestGetResourceInfo_GitHub(c *C) { diff --git a/pkg/validators/validators.go b/pkg/validators/validators.go new file mode 100644 index 0000000000..3fb4cca99c --- /dev/null +++ b/pkg/validators/validators.go @@ -0,0 +1,127 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validators + +import ( + "context" + "fmt" + + compute "cloud.google.com/go/compute/apiv1" + resourcemanager "cloud.google.com/go/resourcemanager/apiv3" + computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" + resourcemanagerpb "google.golang.org/genproto/googleapis/cloud/resourcemanager/v3" +) + +const projectError = "project ID %s does not exist or your credentials do not have permission to access it" +const regionError = "region %s is not available in project ID %s or your credentials do not have permission to access it" +const zoneError = "zone %s is not available in project ID %s or your credentials do not have permission to access it" +const zoneInRegionError = "zone %s is not in region %s in project ID %s or your credentials do not have permissions to access it" + +// TestProjectExists whether projectID exists / is accessible with credentials +func TestProjectExists(projectID string) error { + ctx := context.Background() + c, err := resourcemanager.NewProjectsClient(ctx) + if err != nil { + return err + } + defer c.Close() + + req := &resourcemanagerpb.GetProjectRequest{ + Name: "projects/" + projectID, + } + + _, err = c.GetProject(ctx, req) + if err != nil { + return fmt.Errorf(projectError, projectID) + } + + return nil +} + +func getRegion(projectID string, region string) (*computepb.Region, error) { + ctx := context.Background() + c, err := compute.NewRegionsRESTClient(ctx) + if err != nil { + return nil, err + } + defer c.Close() + + req := &computepb.GetRegionRequest{ + Project: projectID, + Region: region, + } + regionObject, err := c.Get(ctx, req) + if err != nil { + return nil, err + } + + return regionObject, nil +} + +// TestRegionExists whether region exists / is accessible with credentials +func TestRegionExists(projectID string, region string) error { + _, err := getRegion(projectID, region) + if err != nil { + return fmt.Errorf(regionError, region, projectID) + } + return nil +} + +func getZone(projectID string, zone string) (*computepb.Zone, error) { + ctx := context.Background() + c, err := compute.NewZonesRESTClient(ctx) + if err != nil { + return nil, err + } + defer c.Close() + + req := &computepb.GetZoneRequest{ + Project: projectID, + Zone: zone, + } + zoneObject, err := c.Get(ctx, req) + if err != nil { + return nil, err + } + + return zoneObject, nil +} + +// TestZoneExists whether zone exists / is accessible with credentials +func TestZoneExists(projectID string, zone string) error { + _, err := getZone(projectID, zone) + if err != nil { + return fmt.Errorf(zoneError, zone, projectID) + } + return nil +} + +// TestZoneInRegion whether zone is in region +func TestZoneInRegion(projectID string, zone string, region string) error { + regionObject, err := getRegion(projectID, region) + if err != nil { + return fmt.Errorf(regionError, region, projectID) + } + zoneObject, err := getZone(projectID, zone) + if err != nil { + return fmt.Errorf(zoneError, zone, projectID) + } + + if *zoneObject.Region != *regionObject.SelfLink { + return fmt.Errorf(zoneInRegionError, zone, region, projectID) + } + + return nil +} diff --git a/resources/.gitignore b/resources/.gitignore deleted file mode 100644 index a91ac02d3f..0000000000 --- a/resources/.gitignore +++ /dev/null @@ -1,44 +0,0 @@ -#### TERRAFORM - -# Local .terraform directories -**/.terraform/* - -# .tfstate files -*.tfstate -*.tfstate.* - -# Crash log files -crash.log - -# Exclude all .tfvars files, which are likely to contain sentitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject -# to change depending on the environment. -# -*.tfvars - -# Ignore override files as they are usually used to override resources locally and so -# are not checked in -override.tf -override.tf.json -*_override.tf -*_override.tf.json - -# Include override files you do wish to add to version control using negated pattern -# -# !example_override.tf - -# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan -# example: *tfplan* - -# Ignore CLI configuration files -.terraformrc -terraform.rc - -# Ignore Terraform lock files sometimes generated during validation -# These are useful for root modules, but not re-usable modules -.terraform.lock.hcl - -#### PACKER -packer-manifest.json -*.auto.pkrvars.hcl diff --git a/resources/README.md b/resources/README.md index 84d00a5173..8c514190d6 100644 --- a/resources/README.md +++ b/resources/README.md @@ -1,11 +1,22 @@ # Resources -This directory contains a set of resources built for the HPC Toolkit. These -resources can be used to define components of an HPC cluster. +This directory contains a set of resources built for the HPC Toolkit. Resources +describe the building blocks of an HPC blueprint. The expected fields in a +resource are listed in more detail below. -## Referring to resources +## Resource Fields -There are some ways of referring to resources from a configuration YAML as below. +### Source (Required) + +The source is a path or URL that points to the source files for a resource. The +actual content of those files is determined by the [kind](#kind-required) of the +resource. + +A source can be a path which may refer to a resource embedded in the `ghpc` +binary or a local file. It can also be a URL pointing to a github path +containing a conforming module. + +#### Embedded Resources Embedded resources are embedded in the ghpc binary during compilation and cannot be edited. To refer to embedded resources, set the source path to @@ -19,6 +30,8 @@ pre-existing-vpc resource. id: network1 ``` +#### Local Resources + Local resources point to a resource in the file system and can easily be edited. They are very useful during resource development. To use a local resource, set the source to a path starting with `/`, `./`, or `../`. For instance, the @@ -30,6 +43,8 @@ following code is using the local pre-existing-vpc resource. id: network1 ``` +#### Github Resources + GitHub resources point to a resource in GitHub. To use a GitHub resource, set the source to a path starting with `github.com` (over HTTPS) or `git@github.com` (over SSH). For instance, the following codes are using the GitHub @@ -51,23 +66,189 @@ Get resource from GitHub over HTTPS: id: network1 ``` +### Kind (Required) + +Kind refers to the way in which a resource is deployed. Currently, kind can be +either `terraform` or `packer`. + +### ID (Required) + +The `id` field is used to uniquely identify and reference a defined resource. +ID's are used in [variables](../examples/README.md#variables) and become the +name of each module when writing terraform resources. They are also used in the +[use](#use) and [outputs](#outputs) lists described just below. + +For terraform resources, the ID will be rendered into the terraform module label +at the top level main.tf file. + +### Settings (May Be Required) + +The settings field is a map that supplies any user-defined variables for each +resource. Settings values can be simple strings, numbers or booleans, but can +also support complex data types like maps and lists of variable depth. These +settings will become the values for the variables defined in either the +`variables.tf` file for Terraform or `variable.pkr.hcl` file for Packer. + +For some resources, there are mandatory variables that must be set, +therefore `settings` is a required field in that case. In many situations, a +combination of sensible defaults, global variables and used resources can +populated all required settings and therefore the settings field can be left out +entirely. + +### Use (Optional) + +The `use` field is a powerful way of linking a resource to one or more other +resources. When a resource "uses" another resource, the outputs of the used +resource are compared to the settings of the current resource. If they have +matching names, and the setting has no explicit value, then it will be set to +the used resource's output. For example, see the following YAML: + +```yaml +resources: +- source: resources/network/vpc + kind: terraform + id: network1 + +- resource: resources/compute/simple-instance + kind: terraform + id: workstation + use: [network1] + settings: + ... +``` + +In this snippet, the simple instance, `workstation`, uses the outputs of vpc +`network1`. + +In this case both `network_self_link` and `subnetwork_self_link` in the +[`workstation` settings](compute/simple-instance/README.md#Inputs) will be set +to `$(network1.network_self_link)` and `$(network1.subnetwork_self_link)` which +refer to the [`network1` outputs](network/vpc/README#Outputs) +of the same names. + +The order of precedence that `ghpc` uses in determining when to infer a setting +value is the following: + +1. Explicitly set in the config by the user +1. Output from a used resource, taken in the order provided in the `use` list +1. Global variable (`vars`) of the same name +1. Default value for the setting + +### Outputs (Optional) + +The `outputs` field allows a resource-level output to be made available at the +resource group level and therefore will be available via `terraform output` in +terraform-based resources groups. This can useful for displaying the IP of a +login node or simply displaying instructions on how to use a resources, as we +have in the +[monitoring dashboard resource](monitoring/dashboard/README.md#Outputs). + ## Common Settings -There are a few common setting names that are consistent accross different -HPC Toolkit resources. This is intentional to allow multiple resources to share -inferred settings from global variables. These variables are listed and -described below. +The following common naming conventions should be used to decrease the verbosity +needed to define a blueprint via YAML. This is intentional to allow multiple +resources to share inferred settings from global variables. For example, if all +resources are to be created in a single region, that region can be defined as a +global variable, which is shared between all resources without an explicit +setting. -* **project_id**: The associated GCP project ID of the project a resource (or - resources) will be created. +* **project_id**: The GCP project ID in which to create the resource. * **deployment_name**: The name of the current deployment of a blueprint. This - can be changed either in the blueprint itself as needed or in the input yaml. + can help to avoid naming conflicts of resources when multiple deployments are + created from the same set of blueprints. * **region**: The GCP - [region](https://cloud.google.com/compute/docs/regions-zones) for the - resource(s) + [region](https://cloud.google.com/compute/docs/regions-zones) the resource + will be created in. * **zone**: The GCP [zone](https://cloud.google.com/compute/docs/regions-zones) - for the resource(s) + the resource will be created in. * **network_name**: The name of the network a resource will use or connect to. +* **labels**: + [Labels](https://cloud.google.com/resource-manager/docs/creating-managing-labels) + added to the resource. In order to include any resource in advanced + monitoring, labels must be exposed. We strongly recommend that all resources + expose this variable. + +## Writing Custom Resources + +Resources are flexible by design, however we do define some best practices when +creating a new resource. + +### Terraform Requirements + +The resource source field must point to a single module. We recommend the +following structure: + +* main.tf file composing the resources using provided variables. +* variables.tf file defining the variables used. +* (Optional) outputs.tf file defining any exported outputs used (if any). +* (Optional) modules directory pointing to submodules needed to create the + resource. + +### General Best Practices + +* Variables for environment-specific values (like project_id) should not be + given defaults. This forces the calling module to provide meaningful values. +* Variables should only have zero-value defaults (like null or empty strings) + where leaving the variable empty is a valid preference which will not be + rejected by the underlying API(s). +* Set good defaults wherever possible. Be opinionated about HPC use cases. +* Follow common variable [naming conventions](#common-settings). + +### Resource Role + +A resource role is a default label applied to resources (ghpc_role), which +conveys what role that resource plays within a larger HPC environment. + +The resources provided with the HPC toolkit have been divided into roles +matching the names of folders in this directory (ex: compute, file-system etc.). +When possible, custom resources should use these roles so that they match other +resources defined by the toolkit. If a custom resource does not fit into these +roles, a new role can be defined. + +A resource’s parent folder will define the resource’s role. Therefore, +regardless of where the resource is located, the resource directory should be +explicitly referenced 2 layers deep, where the top layer refers to the “role” of +that resource. + +If a resource is not defined 2 layers deep and the ghpc_role label has not been +explicitly set in settings, ghpc_role will default to undefined. + +Below we show a few of the resources and their roles (as parent folders). + +```text +resources/ +├── compute +│ └── simple-instance +├── file-system +│ └── filestore +├── network +│ ├── pre-existing-vpc +│ └── vpc +├── packer +│ └── custom-image +├── scripts +│ ├── omnia-install +│ ├── startup-script +│ └── wait-for-startup +└── third-party + ├── compute + ├── file-system + └── scheduler +``` + +### Terraform Coding Standards + +Any Terraform based resources in the HPC Toolkit repo should implement the following standards: + +* terraform-docs is used to generate README files for each resource. +* The first parameter listed under a module should be source (when referring to an external implementation). +* The order for parameters in inputs should be: + * description + * type + * default +* The order for parameters in outputs should be: + * description + * value ## Available Resources @@ -165,4 +346,4 @@ described below. * [**DDN-EXAScaler**](third-party/file-system/DDN-EXAScaler/README.md): Creates a DDN Exascaler lustre]() file system. This resource has - [license costs](https://pantheon.corp.google.com/marketplace/product/ddnstorage/exascaler-cloud). + [license costs](https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud). diff --git a/resources/compute/simple-instance/README.md b/resources/compute/simple-instance/README.md index 006ad078b3..81a31a1ca2 100644 --- a/resources/compute/simple-instance/README.md +++ b/resources/compute/simple-instance/README.md @@ -21,6 +21,46 @@ This creates a cluster of 8 compute VMs named `compute-[0-7]` on the network defined by the `network1` resource. The VMs are of type c2-standard-60 and mount the `homefs` file system resource. +### Placement + +The `placement_policy` variable can be used to control where your VM instances +are physically located relative to each other within a zone. See the official +placement [guide][guide-link] and [api][api-link] documentation. + +[guide-link]: https://cloud.google.com/compute/docs/instances/define-instance-placement +[api-link]: https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement + +Use the following settings for compact placement: + +```yaml + ... + settings: + instance_count: 4 + machine_type: c2-standard-60 + placement_policy: + vm_count: 4 # Note: should match instance count + collocation: "COLLOCATED" + availability_domain_count: null +``` + +Use the following settings for spread placement: + +```yaml + ... + settings: + instance_count: 4 + machine_type: n2-standard-4 + placement_policy: + vm_count: null + collocation: null + availability_domain_count: 2 +``` + +> **_NOTE:_** Due to +> [this open issue](https://github.com/hashicorp/terraform-provider-google/issues/11483), +> it may be required to specify the `vm_count`. Once this issue is resolved, +> `vm_count` will no longer be mandatory. + ## License @@ -44,12 +84,14 @@ limitations under the License. |------|---------| | [terraform](#requirement\_terraform) | >= 0.14.0 | | [google](#requirement\_google) | >= 3.83 | +| [google-beta](#requirement\_google-beta) | >= 3.73 | ## Providers | Name | Version | |------|---------| | [google](#provider\_google) | >= 3.83 | +| [google-beta](#provider\_google-beta) | >= 3.73 | ## Modules @@ -59,14 +101,16 @@ No modules. | Name | Type | |------|------| +| [google-beta_google_compute_instance.compute_vm](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance) | resource | | [google_compute_disk.boot_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_instance.compute_vm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource | +| [google_compute_resource_policy.placement_policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_resource_policy) | resource | | [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | | [deployment\_name](#input\_deployment\_name) | Name of the deployment, used to name the cluster | `string` | n/a | yes | | [disable\_public\_ips](#input\_disable\_public\_ips) | If set to true, instances will not have public IPs | `bool` | `false` | no | | [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | @@ -80,7 +124,8 @@ No modules. | [name\_prefix](#input\_name\_prefix) | Name Prefix | `string` | `null` | no | | [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. | `string` | `"default"` | no | | [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. | `string` | `"MIGRATE"` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy` requires it to be `TERMINATE` | `string` | `null` | no | +| [placement\_policy](#input\_placement\_policy) | Control where your VM instances are physically located relative to each other within a zone. | 
object({
    vm_count                  = number,
    availability_domain_count = number,
    collocation               = string,
  })
| `null` | no | | [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/devstorage.read_only",
    "https://www.googleapis.com/auth/logging.write",
    "https://www.googleapis.com/auth/monitoring.write",
    "https://www.googleapis.com/auth/servicecontrol",
    "https://www.googleapis.com/auth/service.management.readonly",
    "https://www.googleapis.com/auth/trace.append"
  ]
}
| no | | [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | | [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | diff --git a/resources/compute/simple-instance/main.tf b/resources/compute/simple-instance/main.tf index 1b082c88bf..f1fb360663 100644 --- a/resources/compute/simple-instance/main.tf +++ b/resources/compute/simple-instance/main.tf @@ -19,6 +19,22 @@ locals { { startup-script = var.startup_script }) : {} network_storage = var.network_storage != null ? ( { network_storage = jsonencode(var.network_storage) }) : {} + + resource_prefix = var.name_prefix != null ? var.name_prefix : var.deployment_name + + enable_gvnic = var.bandwidth_tier != "not_enabled" + enable_tier_1 = var.bandwidth_tier == "tier_1_enabled" + + # compact_placement : true when placement policy is provided and collocation set; false if unset + compact_placement = try(var.placement_policy.collocation, null) != null + automatic_restart = local.compact_placement ? false : null + on_host_maintenance_from_placement = local.compact_placement ? "TERMINATE" : "MIGRATE" + + on_host_maintenance = ( + var.on_host_maintenance != null + ? var.on_host_maintenance + : local.on_host_maintenance_from_placement + ) } data "google_compute_image" "compute_image" { @@ -29,24 +45,36 @@ data "google_compute_image" "compute_image" { resource "google_compute_disk" "boot_disk" { count = var.instance_count - name = var.name_prefix != null ? ( - "${var.name_prefix}-boot-disk-${count.index}") : ( - "${var.deployment_name}-boot-disk-${count.index}") + name = "${local.resource_prefix}-boot-disk-${count.index}" image = data.google_compute_image.compute_image.self_link type = var.disk_type size = var.disk_size_gb labels = var.labels } +resource "google_compute_resource_policy" "placement_policy" { + count = var.placement_policy != null ? 1 : 0 + name = "${local.resource_prefix}-simple-instance-placement" + group_placement_policy { + vm_count = var.placement_policy.vm_count + availability_domain_count = var.placement_policy.availability_domain_count + collocation = var.placement_policy.collocation + } +} + resource "google_compute_instance" "compute_vm" { + provider = google-beta + count = var.instance_count depends_on = [var.network_self_link, var.network_storage] - name = var.name_prefix != null ? "${var.name_prefix}-${count.index}" : "${var.deployment_name}-${count.index}" + name = "${local.resource_prefix}-${count.index}" machine_type = var.machine_type zone = var.zone + resource_policies = google_compute_resource_policy.placement_policy[*].self_link + labels = var.labels boot_disk { @@ -63,6 +91,11 @@ resource "google_compute_instance" "compute_vm" { network = var.network_self_link subnetwork = var.subnetwork_self_link + nic_type = local.enable_gvnic ? "GVNIC" : null + } + + network_performance_config { + total_egress_bandwidth_tier = local.enable_tier_1 ? "TIER_1" : "DEFAULT" } dynamic "service_account" { @@ -75,7 +108,8 @@ resource "google_compute_instance" "compute_vm" { guest_accelerator = var.guest_accelerator scheduling { - on_host_maintenance = var.on_host_maintenance + on_host_maintenance = local.on_host_maintenance + automatic_restart = local.automatic_restart } metadata = merge(local.network_storage, local.startup_script, var.metadata) diff --git a/resources/compute/simple-instance/variables.tf b/resources/compute/simple-instance/variables.tf index 4f09033e5c..23bf98d70d 100644 --- a/resources/compute/simple-instance/variables.tf +++ b/resources/compute/simple-instance/variables.tf @@ -140,11 +140,37 @@ variable "guest_accelerator" { } variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance." + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy` requires it to be `TERMINATE`" type = string - default = "MIGRATE" + default = null validation { - condition = contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) - error_message = "The on_host_maintenance must be set to MIGRATE or TERMINATE." + condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) + error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." } } + +variable "bandwidth_tier" { + description = <> "/etc/exports" %{ endfor ~} exportfs -r diff --git a/resources/monitoring/dashboard/README.md b/resources/monitoring/dashboard/README.md index 13afc215ce..e6d8e99b30 100644 --- a/resources/monitoring/dashboard/README.md +++ b/resources/monitoring/dashboard/README.md @@ -77,5 +77,7 @@ No modules. ## Outputs -No outputs. +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for accessing the monitoring dashboard | diff --git a/resources/monitoring/dashboard/outputs.tf b/resources/monitoring/dashboard/outputs.tf new file mode 100644 index 0000000000..b7ff35fb0e --- /dev/null +++ b/resources/monitoring/dashboard/outputs.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "instructions" { + description = "Instructions for accessing the monitoring dashboard" + value = <<-EOT + A monitoring dashboard has been created. To view, navigate to the following URL: + https://console.cloud.google.com/monitoring/dashboards/builder${regex("/[0-9a-z-]*$", google_monitoring_dashboard.dashboard.id)} + EOT +} diff --git a/resources/packer/custom-image/README.md b/resources/packer/custom-image/README.md index d7bad151ff..4b3dd285f9 100644 --- a/resources/packer/custom-image/README.md +++ b/resources/packer/custom-image/README.md @@ -89,17 +89,18 @@ No resources. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [ansible\_playbooks](#input\_ansible\_playbooks) | n/a | 
list(object({
    playbook_file   = string
    galaxy_file     = string
    extra_arguments = list(string)
  }))
| `[]` | no | +| [deployment\_name](#input\_deployment\_name) | HPC Toolkit deployment name | `string` | n/a | yes | | [disk\_size](#input\_disk\_size) | Size of disk image in GB | `number` | `null` | no | -| [machine\_type](#input\_machine\_type) | VM machine type on which to build new image | `string` | `"n2d-standard-4"` | no | +| [machine\_type](#input\_machine\_type) | VM machine type on which to build new image | `string` | `"n2-standard-4"` | no | | [omit\_external\_ip](#input\_omit\_external\_ip) | Provision the image building VM without a public IP address | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | n/a | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which to create VM and image | `string` | n/a | yes | | [service\_account\_email](#input\_service\_account\_email) | The service account email to use. If null or 'default', then the default Compute Engine service account will be used. | `string` | `null` | no | | [service\_account\_scopes](#input\_service\_account\_scopes) | Service account scopes to attach to the instance. See
https://cloud.google.com/compute/docs/access/service-accounts. | `list(string)` | `null` | no | | [source\_image](#input\_source\_image) | Source OS image to build from | `string` | `null` | no | | [source\_image\_family](#input\_source\_image\_family) | Alternative to source\_image. Specify image family to build from latest image in family | `string` | `"hpc-centos-7"` | no | | [source\_image\_project\_id](#input\_source\_image\_project\_id) | A list of project IDs to search for the source image. Packer will search the
first project ID in the list first, and fall back to the next in the list,
until it finds the source image. | `list(string)` | 
[
  "cloud-hpc-image-public"
]
| no | | [ssh\_username](#input\_ssh\_username) | Username to use for SSH access to VM | `string` | `"packer"` | no | -| [subnetwork](#input\_subnetwork) | Name of subnetwork in which to provision image building VM | `string` | n/a | yes | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of subnetwork in which to provision image building VM | `string` | `null` | no | | [tags](#input\_tags) | Assign network tags to apply firewall rules to VM instance | `list(string)` | `null` | no | | [use\_iap](#input\_use\_iap) | Use IAP proxy when connecting by SSH | `bool` | `false` | no | | [use\_os\_login](#input\_use\_os\_login) | Use OS Login when connecting by SSH | `bool` | `false` | no | diff --git a/resources/packer/custom-image/image.pkr.hcl b/resources/packer/custom-image/image.pkr.hcl index bc837382ad..fc95bc5664 100644 --- a/resources/packer/custom-image/image.pkr.hcl +++ b/resources/packer/custom-image/image.pkr.hcl @@ -13,18 +13,18 @@ // limitations under the License. locals { - toolkit_venv = "/usr/local/toolkit" + subnetwork_name = var.subnetwork_name != null ? var.subnetwork_name : "${var.deployment_name}-primary-subnet" } source "googlecompute" "hpc_centos_7" { project_id = var.project_id - image_name = "example-${formatdate("YYYYMMDD't'hhmmss'z'", timestamp())}" - image_family = "example-v1" + image_name = "${var.deployment_name}-${formatdate("YYYYMMDD't'hhmmss'z'", timestamp())}" + image_family = var.deployment_name machine_type = var.machine_type disk_size = var.disk_size omit_external_ip = var.omit_external_ip use_internal_ip = var.omit_external_ip - subnetwork = var.subnetwork + subnetwork = local.subnetwork_name source_image = var.source_image source_image_family = var.source_image_family source_image_project_id = var.source_image_project_id diff --git a/resources/packer/custom-image/variables.pkr.hcl b/resources/packer/custom-image/variables.pkr.hcl index fd88574991..16d3d152fe 100644 --- a/resources/packer/custom-image/variables.pkr.hcl +++ b/resources/packer/custom-image/variables.pkr.hcl @@ -12,14 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. +variable "deployment_name" { + description = "HPC Toolkit deployment name" + type = string +} + variable "project_id" { - type = string + description = "Project in which to create VM and image" + type = string } variable "machine_type" { description = "VM machine type on which to build new image" type = string - default = "n2d-standard-4" + default = "n2-standard-4" } variable "disk_size" { @@ -33,9 +39,10 @@ variable "zone" { type = string } -variable "subnetwork" { +variable "subnetwork_name" { description = "Name of subnetwork in which to provision image building VM" type = string + default = null } variable "omit_external_ip" { diff --git a/resources/scripts/spack-install/README.md b/resources/scripts/spack-install/README.md index 3e921d72e3..6d59899a54 100644 --- a/resources/scripts/spack-install/README.md +++ b/resources/scripts/spack-install/README.md @@ -26,6 +26,10 @@ additional packages, but cannot be used to uninstall packages from the VM. **Please note**: Currently, license installation is performed by copying a license file from a GCS bucket to a specific directory on the target VM. +**Please note**: When populating a buildcache with packages, the VM this +spack resource is running on requires the following scope: +https://www.googleapis.com/auth/devstorage.read_write + ## Example As an example, the below is a possible definition of a spack installation. @@ -43,8 +47,14 @@ As an example, the below is a possible definition of a spack installation. mirror_url: gs://example-buildcache/linux-centos7 configs: - type: 'single-config' - value: 'config:build_tree:/apps/spack/build_stage' + value: 'config:install_tree:/apps/spack/opt' + scope: 'site' + - type: 'file' scope: 'site' + value: | + config: + build_stage: + - /apps/spack/stage - type: 'file' scope: 'site' value: | @@ -146,9 +156,11 @@ No resources. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [caches\_to\_populate](#input\_caches\_to\_populate) | Defines caches which will be populated with the installed packages.
Each cache must specify a type (either directory, or mirror).
Each cache must also specify a path. For directory caches, this path
must be on a local file system (i.e. file:///path/to/cache). For
mirror paths, this can be any valid URL that spack accepts.

NOTE: GPG Keys should be installed before trying to populate a cache
with packages.

NOTE: The gpg\_keys variable can be used to install existing GPG keys
and create new GPG keys, both of which are acceptable for populating a
cache. | `list(map(any))` | `[]` | no | | [compilers](#input\_compilers) | Defines compilers for spack to install before installing packages. | `list(string)` | `[]` | no | | [configs](#input\_configs) | List of configuration options to set within spack.
Configs can be of type 'single-config' or 'file'.
All configs must specify a value, and a
a scope. | `list(map(any))` | `[]` | no | | [environments](#input\_environments) | Defines a spack environment to configure. | 
list(object({
    name     = string
    packages = list(string)
  }))
| `null` | no | +| [gpg\_keys](#input\_gpg\_keys) | GPG Keys to trust within spack.
Each key must define a type. Valid types are 'file' and 'new'.
Keys of type 'file' must define a path to the key that
should be trusted.
Keys of type 'new' must define a 'name' and 'email' to create
the key with. | `list(map(any))` | `[]` | no | | [install\_dir](#input\_install\_dir) | Directory to install spack into. | `string` | `"/apps/spack"` | no | | [licenses](#input\_licenses) | List of software licenses to install within spack. | 
list(object({
    source = string
    dest   = string
  }))
| `null` | no | | [log\_file](#input\_log\_file) | Defines the logfile that script output will be written to | `string` | `"/dev/null"` | no | @@ -164,5 +176,7 @@ No resources. | Name | Description | |------|-------------| | [controller\_startup\_script](#output\_controller\_startup\_script) | Path to the Spack installation script, duplicate for SLURM controller. | +| [install\_spack\_deps\_runner](#output\_install\_spack\_deps\_runner) | Runner to install dependencies for spack using startup-scripts, requires ansible. | +| [install\_spack\_runner](#output\_install\_spack\_runner) | Runner to install Spack using startup-scripts | | [startup\_script](#output\_startup\_script) | Path to the Spack installation script. | diff --git a/resources/scripts/spack-install/main.tf b/resources/scripts/spack-install/main.tf index 8b305288e1..888357f548 100644 --- a/resources/scripts/spack-install/main.tf +++ b/resources/scripts/spack-install/main.tf @@ -18,18 +18,30 @@ locals { script_content = templatefile( "${path.module}/templates/install_spack.tpl", { - ZONE = var.zone - PROJECT_ID = var.project_id - INSTALL_DIR = var.install_dir - SPACK_URL = var.spack_url - SPACK_REF = var.spack_ref - COMPILERS = var.compilers == null ? [] : var.compilers - CONFIGS = var.configs == null ? [] : var.configs - LICENSES = var.licenses == null ? [] : var.licenses - PACKAGES = var.packages == null ? [] : var.packages - ENVIRONMENTS = var.environments == null ? [] : var.environments - MIRRORS = var.spack_cache_url == null ? [] : var.spack_cache_url - LOG_FILE = var.log_file == null ? "/dev/null" : var.log_file + ZONE = var.zone + PROJECT_ID = var.project_id + INSTALL_DIR = var.install_dir + SPACK_URL = var.spack_url + SPACK_REF = var.spack_ref + COMPILERS = var.compilers == null ? [] : var.compilers + CONFIGS = var.configs == null ? [] : var.configs + LICENSES = var.licenses == null ? [] : var.licenses + PACKAGES = var.packages == null ? [] : var.packages + ENVIRONMENTS = var.environments == null ? [] : var.environments + MIRRORS = var.spack_cache_url == null ? [] : var.spack_cache_url + GPG_KEYS = var.gpg_keys == null ? [] : var.gpg_keys + CACHES_TO_POPULATE = var.caches_to_populate == null ? [] : var.caches_to_populate + LOG_FILE = var.log_file == null ? "/dev/null" : var.log_file } ) + install_spack_deps_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/install_spack_deps.yml" + "destination" = "install_spack_deps.yml" + } + install_spack_runner = { + "type" = "shell" + "content" = local.script_content + "destination" = "install_spack.sh" + } } diff --git a/resources/scripts/spack-install/outputs.tf b/resources/scripts/spack-install/outputs.tf index 899a194d61..ca0ebe7b05 100644 --- a/resources/scripts/spack-install/outputs.tf +++ b/resources/scripts/spack-install/outputs.tf @@ -23,3 +23,13 @@ output "controller_startup_script" { description = "Path to the Spack installation script, duplicate for SLURM controller." value = local.script_content } + +output "install_spack_deps_runner" { + description = "Runner to install dependencies for spack using startup-scripts, requires ansible." + value = local.install_spack_deps_runner +} + +output "install_spack_runner" { + description = "Runner to install Spack using startup-scripts" + value = local.install_spack_runner +} diff --git a/resources/scripts/spack-install/templates/install_spack.tpl b/resources/scripts/spack-install/templates/install_spack.tpl index 145c05ea4b..476ac345e3 100755 --- a/resources/scripts/spack-install/templates/install_spack.tpl +++ b/resources/scripts/spack-install/templates/install_spack.tpl @@ -52,6 +52,18 @@ EOF spack mirror add --scope site ${m.mirror_name} ${m.mirror_url} >> ${LOG_FILE} 2>&1 %{endfor ~} + echo "$PREFIX Installing GPG keys" + spack gpg init >> ${LOG_FILE} 2>&1 + %{for k in GPG_KEYS ~} + %{if k.type == "file" ~} + spack gpg trust ${k.path} + %{endif ~} + + %{if k.type == "new" ~} + spack gpg create "${k.name}" ${k.email} + %{endif ~} + %{endfor ~} + spack buildcache keys --install --trust >> ${LOG_FILE} 2>&1 else source ${INSTALL_DIR}/share/spack/setup-env.sh >> ${LOG_FILE} 2>&1 @@ -59,48 +71,65 @@ fi echo "$PREFIX Installing licenses..." %{for lic in LICENSES ~} -gsutil cp ${lic.source} ${lic.dest} >> ${LOG_FILE} 2>&1 + gsutil cp ${lic.source} ${lic.dest} >> ${LOG_FILE} 2>&1 %{endfor ~} echo "$PREFIX Installing compilers..." %{for c in COMPILERS ~} -{ -spack install ${c}; -spack load ${c}; -spack clean -s -} &>> ${LOG_FILE} + { + spack install ${c}; + spack load ${c}; + spack clean -s + } &>> ${LOG_FILE} %{endfor ~} spack compiler find --scope site >> ${LOG_FILE} 2>&1 echo "$PREFIX Installing root spack specs..." %{for p in PACKAGES ~} -spack install ${p} >> ${LOG_FILE} 2>&1 -spack clean -s + spack install ${p} >> ${LOG_FILE} 2>&1 + spack clean -s %{endfor ~} echo "$PREFIX Configuring spack environments" %{for e in ENVIRONMENTS ~} + { + spack env create ${e.name}; + spack env activate ${e.name}; + } &>> ${LOG_FILE} -{ -spack env create ${e.name}; -spack env activate ${e.name}; -} &>> ${LOG_FILE} - -echo "$PREFIX Configuring spack environment ${e.name}" -%{for p in e.packages ~} -spack add ${p} >> ${LOG_FILE} 2>&1 -%{endfor ~} + echo "$PREFIX Configuring spack environment ${e.name}" + %{for p in e.packages ~} + spack add ${p} >> ${LOG_FILE} 2>&1 + %{endfor ~} -echo "$PREFIX Concretizing spack environment ${e.name}" -spack concretize >> ${LOG_FILE} 2>&1 -echo "$PREFIX Installing packages for spack environment ${e.name}" -spack install >> ${LOG_FILE} 2>&1 + echo "$PREFIX Concretizing spack environment ${e.name}" + spack concretize >> ${LOG_FILE} 2>&1 + echo "$PREFIX Installing packages for spack environment ${e.name}" + spack install >> ${LOG_FILE} 2>&1 -spack env deactivate >> ${LOG_FILE} 2>&1 -spack clean -s + spack env deactivate >> ${LOG_FILE} 2>&1 + spack clean -s +%{endfor ~} +echo "$PREFIX Populating defined buildcaches" +%{for c in CACHES_TO_POPULATE ~} + %{if c.type == "directory" ~} + # shellcheck disable=SC2046 + spack buildcache create -d ${c.path} -af $(spack find --format /{hash}) + spack gpg publish -d ${c.path} + spack buildcache update-index -d ${c.path} --keys + %{endif ~} + %{if c.type == "mirror" ~} + # shellcheck disable=SC2046 + spack buildcache create --mirror-url ${c.path} -af $(spack find --format /{hash}) + spack gpg publish --mirror-url ${c.path} + spack buildcache update-index -mirror-url ${c.path} --keys + %{endif ~} %{endfor ~} +echo "source ${INSTALL_DIR}/share/spack/setup-env.sh" >> /etc/profile.d/spack.sh +chmod a+rx /etc/profile.d/spack.sh + echo "$PREFIX Setup complete..." exit 0 diff --git a/resources/scripts/spack-install/variables.tf b/resources/scripts/spack-install/variables.tf index 7869fbdd89..b08ea473fe 100644 --- a/resources/scripts/spack-install/variables.tf +++ b/resources/scripts/spack-install/variables.tf @@ -107,6 +107,75 @@ variable "packages" { type = list(string) } +variable "gpg_keys" { + description = <&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" + exit 1 +} + +handle_debian() { + is_legacy_monitoring_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_MONITORING_PACKAGE} | + grep "${LEGACY_MONITORING_PACKAGE} is installed" + } + + is_legacy_logging_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_LOGGING_PACKAGE} | + grep "${LEGACY_LOGGING_PACKAGE} is installed" + } + + is_legacy_installed() { + is_legacy_monitoring_installed || is_legacy_logging_installed + } + + is_opsagent_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${OPSAGENT_PACKAGE} | + grep "${OPSAGENT_PACKAGE} is installed" + } + + install_opsagent() { + curl -s https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh | bash -s -- --also-install + } +} + +handle_redhat() { + is_legacy_monitoring_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_MONITORING_PACKAGE} | + grep "${LEGACY_MONITORING_PACKAGE} is installed" + } + + is_legacy_logging_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_LOGGING_PACKAGE} | + grep "${LEGACY_LOGGING_PACKAGE} is installed" + } + + is_legacy_installed() { + is_legacy_monitoring_installed || is_legacy_logging_installed + } + + is_opsagent_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${OPSAGENT_PACKAGE} | + grep "${OPSAGENT_PACKAGE} is installed" + } + + install_opsagent() { + curl -s https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh | bash -s -- --also-install + } +} + +main() { + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + handle_redhat + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + handle_debian + else + fail "Unsupported platform." + fi + + if is_legacy_installed || is_opsagent_installed; then + fail "Legacy or Ops Agent is already installed." + fi + + install_opsagent +} + +main diff --git a/resources/scripts/wait-for-startup/README.md b/resources/scripts/wait-for-startup/README.md index 5ce93b84cf..7d4032f1ab 100644 --- a/resources/scripts/wait-for-startup/README.md +++ b/resources/scripts/wait-for-startup/README.md @@ -68,6 +68,7 @@ No modules. |------|-------------|------|---------|:--------:| | [instance\_name](#input\_instance\_name) | Name of the instance we are waiting for | `string` | n/a | yes | | [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [timeout](#input\_timeout) | Timeout in seconds | `number` | `1200` | no | | [zone](#input\_zone) | The GCP zone where the instance is running | `string` | n/a | yes | ## Outputs diff --git a/resources/scripts/wait-for-startup/main.tf b/resources/scripts/wait-for-startup/main.tf index ca64f4ec26..7a0617c60d 100644 --- a/resources/scripts/wait-for-startup/main.tf +++ b/resources/scripts/wait-for-startup/main.tf @@ -13,14 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +locals { + retries = var.timeout / 5 +} resource "null_resource" "wait_for_startup" { provisioner "local-exec" { - command = "${path.module}/scripts/wait-for-startup-status.sh" + command = "/bin/bash ${path.module}/scripts/wait-for-startup-status.sh" environment = { INSTANCE_NAME = var.instance_name ZONE = var.zone PROJECT_ID = var.project_id + RETRIES = local.retries } } } diff --git a/resources/scripts/wait-for-startup/scripts/wait-for-startup-status.sh b/resources/scripts/wait-for-startup/scripts/wait-for-startup-status.sh index 5f9434affd..1d2e9b1870 100755 --- a/resources/scripts/wait-for-startup/scripts/wait-for-startup-status.sh +++ b/resources/scripts/wait-for-startup/scripts/wait-for-startup-status.sh @@ -27,7 +27,7 @@ if [ -z "${PROJECT_ID}" ]; then fi tries=0 -until [ $tries -ge 120 ]; do +until [ $tries -ge "${RETRIES}" ]; do GCLOUD="gcloud compute instances get-serial-port-output ${INSTANCE_NAME} --port 1 --zone ${ZONE} --project ${PROJECT_ID}" FINISH_LINE="startup-script exit status" STATUS_LINE=$(${GCLOUD} 2>/dev/null | grep "${FINISH_LINE}") @@ -45,6 +45,8 @@ elif [ "${STATUS}" == 1 ]; then echo "${GCLOUD}" else echo "invalid return status '${STATUS}'" + echo "to inspect the startup script output, please run:" + echo "${GCLOUD}" exit 1 fi diff --git a/resources/scripts/wait-for-startup/variables.tf b/resources/scripts/wait-for-startup/variables.tf index 90bd0112ab..a464711838 100644 --- a/resources/scripts/wait-for-startup/variables.tf +++ b/resources/scripts/wait-for-startup/variables.tf @@ -28,3 +28,9 @@ variable "project_id" { description = "Project in which the HPC deployment will be created" type = string } + +variable "timeout" { + description = "Timeout in seconds" + type = number + default = 1200 +} diff --git a/resources/third-party/compute/SchedMD-slurm-on-gcp-partition/README.md b/resources/third-party/compute/SchedMD-slurm-on-gcp-partition/README.md index 78b0f54fe2..32c7c1a425 100644 --- a/resources/third-party/compute/SchedMD-slurm-on-gcp-partition/README.md +++ b/resources/third-party/compute/SchedMD-slurm-on-gcp-partition/README.md @@ -51,7 +51,7 @@ No resources. | [compute\_disk\_type](#input\_compute\_disk\_type) | Type of boot disk to create for the partition compute nodes | `string` | `"pd-standard"` | no | | [cpu\_platform](#input\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | | [enable\_placement](#input\_enable\_placement) | Enable placement groups | `bool` | `true` | no | -| [exclusive](#input\_exclusive) | Exclusive job access to nodes | `bool` | `false` | no | +| [exclusive](#input\_exclusive) | Exclusive job access to nodes | `bool` | `true` | no | | [gpu\_count](#input\_gpu\_count) | Number of GPUs attached to the partition compute instances | `number` | `0` | no | | [gpu\_type](#input\_gpu\_type) | Type of GPUs attached to the partition compute instances | `string` | `null` | no | | [image](#input\_image) | Image to be used of the compute VMs in this partition | `string` | `"projects/schedmd-slurm-public/global/images/family/schedmd-slurm-21-08-4-hpc-centos-7"` | no | diff --git a/resources/third-party/compute/SchedMD-slurm-on-gcp-partition/variables.tf b/resources/third-party/compute/SchedMD-slurm-on-gcp-partition/variables.tf index 1e1bc62437..a3437f5cf5 100644 --- a/resources/third-party/compute/SchedMD-slurm-on-gcp-partition/variables.tf +++ b/resources/third-party/compute/SchedMD-slurm-on-gcp-partition/variables.tf @@ -115,7 +115,7 @@ variable "subnetwork_name" { variable "exclusive" { description = "Exclusive job access to nodes" type = bool - default = false + default = true } variable "enable_placement" { diff --git a/resources/third-party/file-system/DDN-EXAScaler/README.md b/resources/third-party/file-system/DDN-EXAScaler/README.md index bbb8bc1784..cb29a0108b 100644 --- a/resources/third-party/file-system/DDN-EXAScaler/README.md +++ b/resources/third-party/file-system/DDN-EXAScaler/README.md @@ -49,7 +49,7 @@ No providers. | Name | Source | Version | |------|--------|---------| -| [ddn\_exascaler](#module\_ddn\_exascaler) | github.com/DDNStorage/exascaler-cloud-terraform//gcp | f329c21 | +| [ddn\_exascaler](#module\_ddn\_exascaler) | github.com/DDNStorage/exascaler-cloud-terraform//gcp | 76ab7fc | ## Resources @@ -64,6 +64,7 @@ No resources. | [clt](#input\_clt) | Compute client target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 0,
  "disk_size": 256,
  "disk_type": "pd-standard"
}
| no | | [fsname](#input\_fsname) | EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long | `string` | `"exacloud"` | no | | [image](#input\_image) | Source image properties | 
object({
    project = string
    name    = string
  })
| 
{
  "name": "exascaler-cloud-v523-centos7",
  "project": "ddn-public"
}
| no | +| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. List of key key, value pairs. | `any` | `{}` | no | | [local\_mount](#input\_local\_mount) | Mountpoint (at the client instances) for this EXAScaler system | `string` | `"/shared"` | no | | [mds](#input\_mds) | Metadata server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | | [mdt](#input\_mdt) | Metadata target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | @@ -74,6 +75,7 @@ No resources. | [network\_self\_link](#input\_network\_self\_link) | The self-link of the VPC network to where the system is connected. Ignored if 'network\_properties' is provided. 'network\_self\_link' or 'network\_properties' must be provided. | `string` | `null` | no | | [oss](#input\_oss) | Object Storage server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 3,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-16",
  "public_ip": true
}
| no | | [ost](#input\_ost) | Object Storage target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | +| [prefix](#input\_prefix) | EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud') | `string` | `null` | no | | [project\_id](#input\_project\_id) | Compute Platform project that will host the EXAScaler filesystem | `string` | n/a | yes | | [security](#input\_security) | Security options | 
object({
    admin              = string
    public_key         = string
    block_project_keys = bool
    enable_os_login    = bool
    enable_local       = bool
    enable_ssh         = bool
    enable_http        = bool
    ssh_source_ranges  = list(string)
    http_source_ranges = list(string)
  })
| 
{
  "admin": "stack",
  "block_project_keys": false,
  "enable_http": false,
  "enable_local": false,
  "enable_os_login": true,
  "enable_ssh": false,
  "http_source_ranges": [
    "0.0.0.0/0"
  ],
  "public_key": null,
  "ssh_source_ranges": [
    "0.0.0.0/0"
  ]
}
| no | | [service\_account](#input\_service\_account) | Service account name used by deploy application | 
object({
    new   = bool
    email = string
  })
| 
{
  "email": null,
  "new": false
}
| no | diff --git a/resources/third-party/file-system/DDN-EXAScaler/main.tf b/resources/third-party/file-system/DDN-EXAScaler/main.tf index 3f031380d2..48e95616b4 100644 --- a/resources/third-party/file-system/DDN-EXAScaler/main.tf +++ b/resources/third-party/file-system/DDN-EXAScaler/main.tf @@ -36,10 +36,12 @@ locals { } module "ddn_exascaler" { - source = "github.com/DDNStorage/exascaler-cloud-terraform//gcp?ref=f329c21" + source = "github.com/DDNStorage/exascaler-cloud-terraform//gcp?ref=76ab7fc" fsname = var.fsname zone = var.zone project = var.project_id + prefix = var.prefix + labels = var.labels security = var.security service_account = var.service_account waiter = var.waiter diff --git a/resources/third-party/file-system/DDN-EXAScaler/variables.tf b/resources/third-party/file-system/DDN-EXAScaler/variables.tf index 536430a7e6..1ffed5dc62 100644 --- a/resources/third-party/file-system/DDN-EXAScaler/variables.tf +++ b/resources/third-party/file-system/DDN-EXAScaler/variables.tf @@ -445,3 +445,15 @@ variable "local_mount" { type = string default = "/shared" } + +variable "prefix" { + description = "EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud')" + type = string + default = null +} + +variable "labels" { + description = "Labels to add to EXAScaler Cloud deployment. List of key key, value pairs." + type = any + default = {} +} diff --git a/tools/cloud-build/Dockerfile b/tools/cloud-build/Dockerfile index ae25ebf4fd..bb814f8b72 100644 --- a/tools/cloud-build/Dockerfile +++ b/tools/cloud-build/Dockerfile @@ -21,7 +21,7 @@ RUN curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - && \ dnsutils \ shellcheck && \ apt-add-repository "deb [arch=$(dpkg --print-architecture)] https://apt.releases.hashicorp.com bullseye main" && \ - apt-get -y update && apt-get install -y unzip python3-pip terraform packer && \ + apt-get -y update && apt-get install -y unzip python3-pip terraform packer jq && \ echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \ diff --git a/tools/cloud-build/daily-tests/README.md b/tools/cloud-build/daily-tests/README.md new file mode 100644 index 0000000000..26c4660f7b --- /dev/null +++ b/tools/cloud-build/daily-tests/README.md @@ -0,0 +1,18 @@ +# Daily and integration tests for the toolkit + +Integration tests have been broken into multiple steps. This allows easily adding new integration tests as build steps under hpc-toolkit-integration-tests. + +Cloud build calls ansible-playbook `[slurm-integration-tests | basic-integration-tests]` with a custom configuration yaml. +Each test has its own yaml under tools/cloud-build/daily-tests/tests. +This file specifies common variables and a list of post_deploy_test, +which can be an empty array for tests that only validate deployment. +Or can list various extra tasks (only one implemented now: `test-mounts-and-partitions`). +This file also specifies the blueprint to create the HPC environment + +The integration test yml, either `slurm-integration-tests` or `basic-integration-tests`, under ansible_playbooks, in turn calls the creation of the blueprint (create_blueprints.sh) and the post_deploy_tests. + +To run the tests on your own project, with your own files, use: + +```shell +gcloud builds submit --config tools/cloud-build/daily-tests/hpc-toolkit-integration-tests.yaml +``` diff --git a/tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml b/tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml new file mode 100644 index 0000000000..d12f880b9d --- /dev/null +++ b/tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml @@ -0,0 +1,169 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: "Setup Integration tests for HPC toolkit" + hosts: localhost + vars: + scripts_dir: "{{ workspace }}/tools/cloud-build/daily-tests" + tasks: + ## Create SSH Keys + - name: "Create .ssh folder" + file: + path: "/builder/home/.ssh" + state: directory + mode: 0700 + - name: Create SSH Key + openssh_keypair: + path: "/builder/home/.ssh/id_rsa" + + ## Create cluster + - name: Create blueprint + command: "{{ scripts_dir }}/create_blueprint.sh" + environment: + EXAMPLE_YAML: "{{ blueprint_yaml }}" + PROJECT_ID: "{{ project }}" + ROOT_DIR: "{{ workspace }}" + BLUEPRINT_DIR: "{{ blueprint_dir }}" + DEPLOYMENT_NAME: "{{ deployment_name }}" + NETWORK: "{{ network }}" + args: + creates: "{{ workspace }}/{{ blueprint_dir }}.tgz" + - name: Create Infrastructure and test + block: + - name: Create Cluster with Terraform + command: + cmd: "{{ item }}" + chdir: "{{ workspace }}/{{ blueprint_dir }}/primary" + args: + creates: "{{ workspace }}/{{ blueprint_dir }}/.terraform" + environment: + TF_IN_AUTOMATION: "TRUE" + with_items: + - "terraform init" + - "terraform apply -auto-approve -no-color" + - name: Get remote IP + register: remote_ip + command: >- + gcloud compute instances describe --zone={{ zone }} {{ remote_node }} + --format='get(networkInterfaces[0].accessConfigs[0].natIP)' + + ## Setup firewall for cloud build + - name: Get Builder IP + shell: >- + dig TXT +short o-o.myaddr.l.google.com @ns1.google.com | + awk -F'"' '{print $2}' + register: build_ip + - name: Create firewall rule + command: + argv: + - gcloud + - compute + - --project={{ project }} + - firewall-rules + - create + - "{{ deployment_name }}" + - --direction=INGRESS + - --priority=1000 + - --network={{ network }} + - --action=ALLOW + - --rules=tcp:22 + - --source-ranges={{ build_ip.stdout }} + - name: 'Add SSH Keys to OS-Login' + command: + argv: + - gcloud + - compute + - os-login + - ssh-keys + - add + - --ttl + - 2h + - "--key-file=/builder/home/.ssh/id_rsa.pub" + - name: Add Remote node as host + add_host: + hostname: "{{ remote_ip.stdout }}" + groups: [remote_host] + - name: Wait for cluster + wait_for_connection: + + ## Cleanup and fail gracefully + rescue: + - name: Delete Firewall Rule + command: + argv: + - gcloud + - compute + - firewall-rules + - delete + - "{{ deployment_name }}" + ignore_errors: true + - name: Tear Down Cluster + run_once: true + delegate_to: localhost + environment: + TF_IN_AUTOMATION: "TRUE" + command: + cmd: terraform destroy -auto-approve + chdir: "{{ workspace }}/{{ blueprint_dir }}/primary" + - name: Fail Out + fail: + msg: "Failed while setting up test infrastructure" + when: true + +- name: Run Integration Tests + hosts: remote_host + gather_facts: false + tasks: + - name: Remote Test Block + vars: + ansible_ssh_private_key_file: "/builder/home/.ssh/id_rsa" + block: + - name: Pause for 2 minutes to allow cluster setup + pause: + minutes: 2 + - name: Run Integration tests for HPC toolkit + include_tasks: "{{ test }}" + run_once: true + vars: + remote_node: "{{ remote_node }}" + deployment_name: "{{ deployment_name }}" + mounts: "{{ mounts }}" + partitions: "{{ partitions }}" + loop: "{{ post_deploy_tests }}" + loop_control: + loop_var: test + + ## Always cleanup, even on failure + always: + - name: Delete Firewall Rule + run_once: true + delegate_to: localhost + command: + argv: + - gcloud + - compute + - firewall-rules + - delete + - "{{ deployment_name }}" + ignore_errors: true + - name: Tear Down Cluster + run_once: true + delegate_to: localhost + environment: + TF_IN_AUTOMATION: "TRUE" + command: + cmd: terraform destroy -auto-approve + chdir: "{{ workspace }}/{{ blueprint_dir }}/primary" diff --git a/tools/cloud-build/daily-tests/ansible_playbooks/packer-integration-test.yml b/tools/cloud-build/daily-tests/ansible_playbooks/packer-integration-test.yml new file mode 100644 index 0000000000..2de03cee89 --- /dev/null +++ b/tools/cloud-build/daily-tests/ansible_playbooks/packer-integration-test.yml @@ -0,0 +1,68 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: "Packer Integration test for HPC toolkit" + hosts: localhost + vars: + scripts_dir: "{{ workspace }}/tools/cloud-build/daily-tests" + tasks: + ## Create blueprint + - name: Create blueprint + command: "{{ scripts_dir }}/create_blueprint.sh" + environment: + EXAMPLE_YAML: "{{ blueprint_yaml }}" + PROJECT_ID: "{{ project }}" + ROOT_DIR: "{{ workspace }}" + BLUEPRINT_DIR: "{{ blueprint_dir }}" + DEPLOYMENT_NAME: "{{ deployment_name }}" + args: + creates: "{{ workspace }}/{{ blueprint_dir }}.tgz" + - name: Create Infrastructure and test + block: + - name: Create Network with Terraform + command: + cmd: "{{ item }}" + chdir: "{{ workspace }}/{{ blueprint_dir }}/network" + args: + creates: "{{ workspace }}/{{ blueprint_dir }}/.terraform" + environment: + TF_IN_AUTOMATION: "TRUE" + with_items: + - terraform init + - terraform validate + - terraform apply -auto-approve -no-color + - name: Create VM image with Packer + command: + cmd: "{{ item }}" + chdir: "{{ workspace }}/{{ blueprint_dir }}/packer/custom-image" + with_items: + - packer validate . + - packer build . + - name: Delete VM Image + ansible.builtin.shell: | + gcloud compute images delete --project={{ project }} --quiet $(jq -r '.builds[-1].artifact_id' packer-manifest.json | cut -d ":" -f2) + args: + chdir: "{{ workspace }}/{{ blueprint_dir }}/packer/custom-image" + ## Always cleanup network + always: + - name: Tear Down Network + run_once: true + delegate_to: localhost + environment: + TF_IN_AUTOMATION: "TRUE" + command: + cmd: terraform destroy -auto-approve -no-color + chdir: "{{ workspace }}/{{ blueprint_dir }}/network" diff --git a/tools/cloud-build/daily-tests/daily-tests.yml b/tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml similarity index 97% rename from tools/cloud-build/daily-tests/daily-tests.yml rename to tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml index f91a2b77fe..0668b271cf 100644 --- a/tools/cloud-build/daily-tests/daily-tests.yml +++ b/tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml @@ -141,11 +141,15 @@ pause: minutes: 2 - name: Run Integration tests for HPC toolkit - include_tasks: slurm-tests.yml + include_tasks: "{{ test }}" run_once: true vars: + login_node: "{{ login_node }}" mounts: "{{ mounts }}" partitions: "{{ partitions }}" + loop: "{{ post_deploy_tests }}" + loop_control: + loop_var: test ## Always cleanup, even on failure always: diff --git a/tools/cloud-build/daily-tests/ansible_playbooks/spack.yml b/tools/cloud-build/daily-tests/ansible_playbooks/spack.yml new file mode 100644 index 0000000000..66adf38562 --- /dev/null +++ b/tools/cloud-build/daily-tests/ansible_playbooks/spack.yml @@ -0,0 +1,37 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Wait for startup script to complete + become: true + wait_for: + path: /var/log/messages + search_regex: '.*{{ login_node }}.*startup-script exit status ([0-9]+)' + timeout: 7200 + state: present + register: startup_status +- name: Fail if startup script exited with a non-zero return code + fail: + msg: There was a failure in the startup script + when: startup_status['match_groups'][0] != "0" +- name: Ensure spack is installed + command: spack --version +- name: Ensure gromacs is installed + shell: spack load gromacs +- name: Test gromacs is available on compute nodes + shell: | + spack load gromacs + srun -N 1 gmx_mpi -version + sleep 120 diff --git a/tools/cloud-build/daily-tests/ansible_playbooks/test-monitoring.yml b/tools/cloud-build/daily-tests/ansible_playbooks/test-monitoring.yml new file mode 100644 index 0000000000..db81bfc35d --- /dev/null +++ b/tools/cloud-build/daily-tests/ansible_playbooks/test-monitoring.yml @@ -0,0 +1,40 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Wait for startup script to complete + become: true + wait_for: + path: /var/log/messages + search_regex: '.*{{ remote_node }}.*startup-script exit status ([0-9]+)' + register: startup_status +- name: Fail if ops agent is not running + become: true + command: systemctl is-active {{ item }} + with_items: + - google-cloud-ops-agent.service + - google-cloud-ops-agent-fluent-bit.service + - google-cloud-ops-agent-opentelemetry-collector.service +- name: Check that monitoring dashboard has been created + command: gcloud monitoring dashboards list --format="get(displayName)" + run_once: true + delegate_to: localhost + register: dashboards +- debug: + var: dashboards +- name: Fail if the HPC Dashboard hasn't been created + fail: + msg: Failed to create dashboard + when: "deployment_name not in dashboards.stdout" diff --git a/tools/cloud-build/daily-tests/slurm-tests.yml b/tools/cloud-build/daily-tests/ansible_playbooks/test-mounts-and-partitions.yml similarity index 97% rename from tools/cloud-build/daily-tests/slurm-tests.yml rename to tools/cloud-build/daily-tests/ansible_playbooks/test-mounts-and-partitions.yml index 9510f205b5..b13a04504e 100644 --- a/tools/cloud-build/daily-tests/slurm-tests.yml +++ b/tools/cloud-build/daily-tests/ansible_playbooks/test-mounts-and-partitions.yml @@ -1,4 +1,4 @@ -# Copyright 2021 Google LLC +# Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tools/cloud-build/daily-tests/blueprints/monitoring.yaml b/tools/cloud-build/daily-tests/blueprints/monitoring.yaml new file mode 100644 index 0000000000..c8a0e03436 --- /dev/null +++ b/tools/cloud-build/daily-tests/blueprints/monitoring.yaml @@ -0,0 +1,72 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +blueprint_name: blueprint-monitoring + +vars: + project_id: ## Set GCP Project ID Here ## + deployment_name: monitoring + region: us-central1 + zone: us-central1-c + +resource_groups: +- group: primary + resources: + - source: resources/network/vpc + kind: terraform + id: network + settings: + network_name: monitoring-net + + - source: resources/file-system/nfs-server + kind: terraform + id: homefs + use: [network] + settings: + local_mounts: [/home] + auto_delete_disk: true + + - source: ./resources/scripts/startup-script + kind: terraform + id: startup + settings: + runners: + - type: shell + source: modules/startup-script/examples/install_cloud_ops_agent.sh + destination: install_cloud_ops_agent.sh + - type: shell + source: modules/startup-script/examples/install_ansible.sh + destination: install_ansible.sh + - $(homefs.install_nfs_client_runner) + - $(homefs.mount_runner) + + - source: ./resources/compute/simple-instance + kind: terraform + id: workstation + use: + - network + - homefs + - startup + settings: + machine_type: c2-standard-4 + metadata: + enable-oslogin: TRUE + + - source: ./resources/monitoring/dashboard + kind: terraform + id: hpc-dash + settings: + title: $(vars.deployment_name) diff --git a/tools/cloud-build/daily-tests/create_blueprint.sh b/tools/cloud-build/daily-tests/create_blueprint.sh index 30e2fc1f6e..b5c8a10c87 100755 --- a/tools/cloud-build/daily-tests/create_blueprint.sh +++ b/tools/cloud-build/daily-tests/create_blueprint.sh @@ -64,9 +64,16 @@ sed -i "s/network_name: .*/network_name: ${NETWORK}/" "${EXAMPLE_YAML}" || sed -i "s/max_node_count: .*/max_node_count: ${MAX_NODES}/" "${EXAMPLE_YAML}" || { echo "could not set max_node_count" - exit 1 } ## Create blueprint and create artifact -./ghpc create -c "${EXAMPLE_YAML}" -tar -czf "${BLUEPRINT_DIR}.tgz" "${BLUEPRINT_DIR}" +./ghpc create "${EXAMPLE_YAML}" || + { + echo "could not write blueprint" + exit 1 + } +tar -czf "${BLUEPRINT_DIR}.tgz" "${BLUEPRINT_DIR}" || + { + echo "could not tarball blueprint" + exit 1 + } diff --git a/tools/cloud-build/daily-tests/hpc-toolkit-integration-tests.yaml b/tools/cloud-build/daily-tests/hpc-toolkit-integration-tests.yaml deleted file mode 100644 index ac2fa0d6e0..0000000000 --- a/tools/cloud-build/daily-tests/hpc-toolkit-integration-tests.yaml +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -timeout: 7200s -steps: -- name: golang - entrypoint: /bin/bash - args: - - -c - - | - cd /workspace - make -- name: >- - us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder - entrypoint: /bin/bash - env: - - "ANSIBLE_HOST_KEY_CHECKING=false" - - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg" - args: - - -c - - | - BUILD_ID_FULL=$BUILD_ID - DEPLOYMENT_NAME=lustre-new-vpc-$${BUILD_ID_FULL:0:6} - LOGIN_ID=slurm-$${DEPLOYMENT_NAME}-login0 - CONTROLLER_ID=slurm-$${DEPLOYMENT_NAME}-controller - cat < test-vars.yml - deployment_name: $${DEPLOYMENT_NAME} - project: ${PROJECT_ID} - zone: us-central1-c - workspace: /workspace - blueprint_yaml: "{{ workspace }}/tools/cloud-build/daily-tests/blueprints/lustre-with-new-vpc.yaml" - blueprint_dir: blueprint-lustre-new-vcp - network: $${DEPLOYMENT_NAME} - max_nodes: 5 - login_node: $${LOGIN_ID} - controller_node: $${CONTROLLER_ID} - partitions: - - compute - mounts: - - /home - - /scratch - EOT - ansible-playbook tools/cloud-build/daily-tests/daily-tests.yml \ - --user=sa_106486320838376751393 --extra-vars="@test-vars.yml" -- name: >- - us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder - entrypoint: /bin/bash - env: - - "ANSIBLE_HOST_KEY_CHECKING=false" - - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg" - args: - - -c - - | - BUILD_ID_FULL=$BUILD_ID - DEPLOYMENT_NAME=hpc-high-io-$${BUILD_ID_FULL:0:6} - LOGIN_ID=slurm-$${DEPLOYMENT_NAME}-login0 - CONTROLLER_ID=slurm-$${DEPLOYMENT_NAME}-controller - cat < test-vars.yml - deployment_name: $${DEPLOYMENT_NAME} - project: ${PROJECT_ID} - zone: us-central1-c - workspace: /workspace - blueprint_yaml: "{{ workspace }}/examples/hpc-cluster-high-io.yaml" - blueprint_dir: blueprint-cluster-high-io - network: default - max_nodes: 5 - login_node: $${LOGIN_ID} - controller_node: $${CONTROLLER_ID} - partitions: - - compute - mounts: - - /home - - /scratch - - /projects - EOT - ansible-playbook tools/cloud-build/daily-tests/daily-tests.yml \ - --user=sa_106486320838376751393 --extra-vars="@test-vars.yml" diff --git a/tools/cloud-build/daily-tests/integration-group-1.yaml b/tools/cloud-build/daily-tests/integration-group-1.yaml new file mode 100644 index 0000000000..c42d0710bc --- /dev/null +++ b/tools/cloud-build/daily-tests/integration-group-1.yaml @@ -0,0 +1,62 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Current parallel execution tree, built to minimize total execution time and faster tests first. +# ├── build_hpc +# └── fetch_builder +# ├── hpc-high-io (group 1) + + +timeout: 14400s # 4hr +steps: +## Test simple golang build +- id: build_ghpc + waitFor: ["-"] + name: golang + entrypoint: /bin/bash + args: + - -c + - | + cd /workspace + make +- id: fetch_builder + waitFor: ["-"] + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + args: + - -c + - echo "done fetching builder" + +## Test Slurm High IO Example +- id: hpc-high-io + waitFor: + - fetch_builder + - build_ghpc + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + env: + - "ANSIBLE_HOST_KEY_CHECKING=false" + - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg" + args: + - -c + - | + set -x -e + BUILD_ID_FULL=$BUILD_ID + BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6} + + ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml \ + --user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" --extra-vars="@tools/cloud-build/daily-tests/tests/hpc-high-io.yml" diff --git a/tools/cloud-build/daily-tests/integration-group-2.yaml b/tools/cloud-build/daily-tests/integration-group-2.yaml new file mode 100644 index 0000000000..cd6fb58483 --- /dev/null +++ b/tools/cloud-build/daily-tests/integration-group-2.yaml @@ -0,0 +1,72 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Current parallel execution tree, built to minimize total execution time and faster tests first. +# ├── build_hpc +# └── fetch_builder +# ├── spack-gromacks (group 2) + + +timeout: 14400s # 4hr +availableSecrets: + secretManager: + - versionName: projects/$PROJECT_ID/secrets/spack_cache_url/versions/2 + env: SPACK_CACHE +steps: +## Test simple golang build +- id: build_ghpc + waitFor: ["-"] + name: golang + entrypoint: /bin/bash + args: + - -c + - | + cd /workspace + make +- id: fetch_builder + waitFor: ["-"] + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + args: + - -c + - echo "done fetching builder" + +## Test Spack Gromacs Example +- id: spack-gromacs + waitFor: + - fetch_builder + - build_ghpc + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + env: + - "ANSIBLE_HOST_KEY_CHECKING=false" + - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg" + secretEnv: ['SPACK_CACHE'] + args: + - -c + - | + set -x -e + BUILD_ID_FULL=$BUILD_ID + BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6} + + sed -i "s/# spack_cache_url:/spack_cache_url:/" examples/spack-gromacs.yaml + sed -i "s/# - mirror_name: gcs_cache/- mirror_name: gcs_cache/" examples/spack-gromacs.yaml + sed -i "s/# mirror_url: .*/ mirror_url: $${SPACK_CACHE//\//\\\/}/" examples/spack-gromacs.yaml + + ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml \ + --user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" \ + --extra-vars="@tools/cloud-build/daily-tests/tests/spack-gromacs.yml" diff --git a/tools/cloud-build/daily-tests/integration-group-3.yaml b/tools/cloud-build/daily-tests/integration-group-3.yaml new file mode 100644 index 0000000000..cd5fd88124 --- /dev/null +++ b/tools/cloud-build/daily-tests/integration-group-3.yaml @@ -0,0 +1,129 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Current parallel execution tree, built to minimize total execution time and faster tests first. +# ├── build_hpc +# └── fetch_builder +# └── packer (group 3) +# └── monitoring (group 3) +# └── omnia +# └── lustre-new-vpc + + +timeout: 14400s # 4hr +steps: +## Test simple golang build +- id: build_ghpc + waitFor: ["-"] + name: golang + entrypoint: /bin/bash + args: + - -c + - | + cd /workspace + make +- id: fetch_builder + waitFor: ["-"] + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + args: + - -c + - echo "done fetching builder" + +## Test monitoring dashboard and install script +- id: monitoring + waitFor: + - fetch_builder + - build_ghpc + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + env: + - "ANSIBLE_HOST_KEY_CHECKING=false" + - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg" + args: + - -c + - | + set -x -e + BUILD_ID_FULL=$BUILD_ID + BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6} + + ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml \ + --user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" \ + --extra-vars="@tools/cloud-build/daily-tests/tests/monitoring.yml" + +## Test Omnia Example +- id: omnia + waitFor: + - monitoring + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + env: + - "ANSIBLE_HOST_KEY_CHECKING=false" + - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg" + args: + - -c + - | + set -x -e + BUILD_ID_FULL=$BUILD_ID + BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6} + + ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml \ + --user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" \ + --extra-vars="@tools/cloud-build/daily-tests/tests/omnia.yml" + +## Test DDN Lustre with new VPC +- id: lustre-new-vpc + waitFor: + - omnia + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + env: + - "ANSIBLE_HOST_KEY_CHECKING=false" + - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg" + args: + - -c + - | + set -x -e + BUILD_ID_FULL=$BUILD_ID + BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6} + + ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml \ + --user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" --extra-vars="@tools/cloud-build/daily-tests/tests/lustre-new-vpc.yml" + +# test building an image using Packer +- id: packer + waitFor: + - fetch_builder + - build_ghpc + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + env: + - "ANSIBLE_HOST_KEY_CHECKING=false" + - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg" + args: + - -c + - | + set -x -e + BUILD_ID_FULL=$BUILD_ID + BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6} + + ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/packer-integration-test.yml \ + --user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" \ + --extra-vars="@tools/cloud-build/daily-tests/tests/packer.yml" diff --git a/tools/cloud-build/daily-tests/tests/hpc-high-io.yml b/tools/cloud-build/daily-tests/tests/hpc-high-io.yml new file mode 100644 index 0000000000..e1ea6ad647 --- /dev/null +++ b/tools/cloud-build/daily-tests/tests/hpc-high-io.yml @@ -0,0 +1,34 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +deployment_name: "hpc-high-io-{{ build }}" +zone: us-central1-c +workspace: /workspace +blueprint_yaml: "{{ workspace }}/examples/hpc-cluster-high-io.yaml" +blueprint_dir: blueprint-hpc-high-io +network: "default" +max_nodes: 5 +login_node: "slurm-{{ deployment_name }}-login0" +controller_node: "slurm-{{ deployment_name }}-controller" +post_deploy_tests: +- test-mounts-and-partitions.yml +partitions: +- compute +- low_cost +mounts: +- /home +- /scratch +- /projects diff --git a/tools/cloud-build/daily-tests/tests/lustre-new-vpc.yml b/tools/cloud-build/daily-tests/tests/lustre-new-vpc.yml new file mode 100644 index 0000000000..d7e48c3b2a --- /dev/null +++ b/tools/cloud-build/daily-tests/tests/lustre-new-vpc.yml @@ -0,0 +1,32 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +deployment_name: "lustre-new-vpc-{{ build }}" +zone: us-central1-c +workspace: /workspace +blueprint_yaml: "{{ workspace }}/tools/cloud-build/daily-tests/blueprints/lustre-with-new-vpc.yaml" +blueprint_dir: blueprint-lustre-new-vcp +network: "{{deployment_name}}" +max_nodes: 5 +login_node: "slurm-{{ deployment_name }}-login0" +controller_node: "slurm-{{ deployment_name }}-controller" +post_deploy_tests: +- test-mounts-and-partitions.yml +partitions: +- compute +mounts: +- /home +- /scratch diff --git a/tools/cloud-build/daily-tests/tests/monitoring.yml b/tools/cloud-build/daily-tests/tests/monitoring.yml new file mode 100644 index 0000000000..a8e0465bd7 --- /dev/null +++ b/tools/cloud-build/daily-tests/tests/monitoring.yml @@ -0,0 +1,25 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +deployment_name: monitoring-{{ build }} +zone: us-central1-c +workspace: /workspace +blueprint_yaml: "{{ workspace }}/tools/cloud-build/daily-tests/blueprints/monitoring.yaml" +blueprint_dir: blueprint-monitoring +network: "{{ deployment_name }}-net" +remote_node: "{{ deployment_name }}-0" +post_deploy_tests: +- test-monitoring.yml diff --git a/tools/cloud-build/daily-tests/tests/omnia.yml b/tools/cloud-build/daily-tests/tests/omnia.yml new file mode 100644 index 0000000000..37a6b7838e --- /dev/null +++ b/tools/cloud-build/daily-tests/tests/omnia.yml @@ -0,0 +1,24 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +deployment_name: "omnia-{{ build }}" +zone: us-central1-c +workspace: /workspace +blueprint_yaml: "{{ workspace }}/examples/omnia-cluster.yaml" +blueprint_dir: blueprint-omnia +network: "default" +remote_node: "omnia-manager-0" +post_deploy_tests: [] diff --git a/tools/cloud-build/daily-tests/tests/packer.yml b/tools/cloud-build/daily-tests/tests/packer.yml new file mode 100644 index 0000000000..32cd229e5b --- /dev/null +++ b/tools/cloud-build/daily-tests/tests/packer.yml @@ -0,0 +1,21 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +deployment_name: packer-image-{{ build }} +zone: us-central1-c +workspace: /workspace +blueprint_yaml: "{{ workspace }}/examples/image-builder.yaml" +blueprint_dir: image-builder diff --git a/tools/cloud-build/daily-tests/tests/spack-gromacs.yml b/tools/cloud-build/daily-tests/tests/spack-gromacs.yml new file mode 100644 index 0000000000..935fd89eb8 --- /dev/null +++ b/tools/cloud-build/daily-tests/tests/spack-gromacs.yml @@ -0,0 +1,32 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +deployment_name: "spack-gromacs-{{ build }}" +zone: us-central1-c +workspace: /workspace +blueprint_yaml: "{{ workspace }}/examples/spack-gromacs.yaml" +blueprint_dir: spack-gromacs +network: "default" +max_nodes: 5 +login_node: slurm-{{ deployment_name }}-login0 +controller_node: slurm-{{ deployment_name }}-controller +post_deploy_tests: +- spack.yml +partitions: +- compute +mounts: +- /home +- /sw diff --git a/tools/cloud-build/hpc-toolkit-builder.yaml b/tools/cloud-build/hpc-toolkit-builder.yaml index 1cecd3edb8..2ed1e7134a 100644 --- a/tools/cloud-build/hpc-toolkit-builder.yaml +++ b/tools/cloud-build/hpc-toolkit-builder.yaml @@ -15,7 +15,8 @@ --- steps: -- name: 'gcr.io/cloud-builders/docker' +- id: builder + name: 'gcr.io/cloud-builders/docker' args: - 'build' - '-t' @@ -25,34 +26,73 @@ steps: - '-f' - 'tools/cloud-build/Dockerfile' - '.' -- name: 'gcr.io/cloud-builders/docker' +- id: pre-commits-setup + waitFor: + - builder + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + args: + - '-c' + - | + set -e + pre-commit install + tools/cloud-build/split-pre-commit-hooks.sh +- id: make-tests + waitFor: + - pre-commits-setup + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash args: - - 'run' - - '--entrypoint' - - '/bin/bash' - - '--volume' - - '/workspace:/ghpc' - - 'us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder' - '-c' - | set -e export PROJECT=build-project addlicense -check . >/dev/null make tests -- name: 'gcr.io/cloud-builders/docker' +- id: pre-commits1 + waitFor: + - pre-commits-setup + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + args: + - '-c' + - | + set -e -x + time tflint --init + export SKIP=$(cat hooks1.txt) + echo skipping: $$SKIP + time pre-commit run --all-files +- id: pre-commits2 + waitFor: + - pre-commits-setup + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + args: + - '-c' + - | + set -e + time tflint --init + export SKIP=$(cat hooks2.txt) + echo skipping: $$SKIP + time pre-commit run --all-files +- id: pre-commits3 + waitFor: + - pre-commits-setup + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash args: - - 'run' - - '--entrypoint' - - '/bin/bash' - - '--volume' - - '/workspace:/ghpc' - - 'us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder' - '-c' - | set -e - pre-commit install --install-hooks - tflint --init - SKIP=go-unit-tests pre-commit run --all-files + time tflint --init + export SKIP=$(cat hooks3.txt) + echo skipping: $$SKIP + time pre-commit run --all-files images: [ 'us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder'] timeout: "1500s" diff --git a/tools/cloud-build/hpc-toolkit-pr-validation.yaml b/tools/cloud-build/hpc-toolkit-pr-validation.yaml index b7dc157c4b..d38f678d8c 100644 --- a/tools/cloud-build/hpc-toolkit-pr-validation.yaml +++ b/tools/cloud-build/hpc-toolkit-pr-validation.yaml @@ -15,34 +15,71 @@ --- steps: -- name: 'gcr.io/cloud-builders/docker' +- id: pre-commits-setup + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash args: - - 'run' - - '--entrypoint' - - '/bin/bash' - - '--volume' - - '/workspace:/ghpc' - - 'us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder' - '-c' - | set -e - export PROJECT=build-project - time addlicense -check . || { echo "addlicense failed"; exit 1; } - time make tests -- name: 'gcr.io/cloud-builders/docker' + pre-commit install + tools/cloud-build/split-pre-commit-hooks.sh +- id: pre-commits1 + waitFor: + - pre-commits-setup + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + args: + - '-c' + - | + set -e -x + time tflint --init + export SKIP=$(cat hooks1.txt) + echo skipping: $$SKIP + time pre-commit run --all-files +- id: pre-commits2 + waitFor: + - pre-commits-setup + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash args: - - 'run' - - '--entrypoint' - - '/bin/bash' - - '--volume' - - '/workspace:/ghpc' - - 'us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder' - '-c' - | set -e - pre-commit install --install-hooks time tflint --init - time SKIP=go-unit-tests pre-commit run --all-files + export SKIP=$(cat hooks2.txt) + echo skipping: $$SKIP + time pre-commit run --all-files +- id: pre-commits3 + waitFor: + - pre-commits-setup + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + args: + - '-c' + - | + set -e + time tflint --init + export SKIP=$(cat hooks3.txt) + echo skipping: $$SKIP + time pre-commit run --all-files +- id: make-tests + waitFor: + - pre-commits-setup + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + args: + - '-c' + - | + set -e + export PROJECT=build-project + time addlicense -check . || { echo "addlicense failed"; exit 1; } + time make tests timeout: "1200s" options: machineType: N1_HIGHCPU_8 diff --git a/tools/cloud-build/split-pre-commit-hooks.sh b/tools/cloud-build/split-pre-commit-hooks.sh new file mode 100755 index 0000000000..3eabdf10cd --- /dev/null +++ b/tools/cloud-build/split-pre-commit-hooks.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tac .pre-commit-config.yaml | grep id: | cut -d ':' -f2 | sed -e 's/ //g' >hooks.txt +lines=$(wc -l >xaa && cat xaa xab | sort | uniq | xargs | sed -e 's/ /,/g' >hooks1.txt +echo go-unit-tests >>xab && cat xab xac | sort | uniq | xargs | sed -e 's/ /,/g' >hooks2.txt +echo go-unit-tests >>xac && cat xac xaa | sort | uniq | xargs | sed -e 's/ /,/g' >hooks3.txt +echo "created hooks1.txt hooks2.txt and hooks3.txt with three lists of hooks to skp" diff --git a/tools/validate_configs/test_configs/packer.yaml b/tools/validate_configs/test_configs/packer.yaml index 3a0a227be2..2d70d77eb8 100644 --- a/tools/validate_configs/test_configs/packer.yaml +++ b/tools/validate_configs/test_configs/packer.yaml @@ -36,7 +36,6 @@ resource_groups: kind: packer id: my-custom-image settings: - subnetwork: "subnet-central1" use_iap: true omit_external_ip: true disk_size: 100 diff --git a/tools/validate_configs/test_configs/spack-buildcache.yaml b/tools/validate_configs/test_configs/spack-buildcache.yaml new file mode 100644 index 0000000000..7775678203 --- /dev/null +++ b/tools/validate_configs/test_configs/spack-buildcache.yaml @@ -0,0 +1,100 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +blueprint_name: spack-buildcache + +vars: + project_id: ## Set GCP Project ID Here ## + deployment_name: spack-buildcache + region: us-central1 + zone: us-central1-c + +resource_groups: +- group: primary + resources: + - source: resources/network/pre-existing-vpc + kind: terraform + id: network1 + + - source: resources/scripts/spack-install + kind: terraform + id: spack + settings: + install_dir: /apps/spack + spack_url: https://github.com/spack/spack + spack_ref: v0.17.1 + log_file: /var/log/spack.log + configs: + - type: 'single-config' + scope: 'site' + value: 'config:install_tree:padded_length:128' + compilers: + - gcc@10.3.0 target=x86_64 + packages: + - intel-mpi@2018.4.274%gcc@10.3.0 + - gromacs@2021.2 %gcc@10.3.0 ^intel-mpi@2018.4.274 + gpg_keys: + - type: 'file' + path: '/tmp/spack_key.gpg' + caches_to_populate: + - type: 'mirror' + path: ## Add GCS bucket to populate here ## + + - source: resources/scripts/startup-script + kind: terraform + id: spack-startup + settings: + runners: + - type: data + source: ## Add path to GPG key here ## + destination: /tmp/spack_key.gpg + - type: shell + content: | + #!/bin/bash + mkdir /apps + chmod a+rwx /apps + destination: apps_create.sh + - type: shell + source: modules/startup-script/examples/install_ansible.sh + destination: install_ansible.sh + - type: ansible-local + source: modules/spack-install/scripts/install_spack_deps.yml + destination: install_spack_deps.yml + - type: shell + content: $(spack.startup_script) + destination: install_spack.sh + - type: shell + destination: shutdown.sh + content: shutdown -h + + - source: resources/compute/simple-instance + kind: terraform + id: spack-build + use: + - network1 + - spack-startup + settings: + name_prefix: spack-builder + machine_type: n2-standard-8 + service_account: + email: null + scopes: + - "https://www.googleapis.com/auth/devstorage.read_write" + - "https://www.googleapis.com/auth/logging.write" + - "https://www.googleapis.com/auth/monitoring.write" + - "https://www.googleapis.com/auth/servicecontrol" + - "https://www.googleapis.com/auth/service.management.readonly" + - "https://www.googleapis.com/auth/trace.append" diff --git a/tools/validate_configs/validate_configs.sh b/tools/validate_configs/validate_configs.sh index 838791e39d..d07293bb0d 100755 --- a/tools/validate_configs/validate_configs.sh +++ b/tools/validate_configs/validate_configs.sh @@ -38,7 +38,7 @@ run_test() { exit 1 } cd "${cwd}" - ./ghpc create "${tmpdir}"/"${exampleFile}" >/dev/null || + ./ghpc create -l IGNORE "${tmpdir}"/"${exampleFile}" >/dev/null || { echo "*** ERROR: error creating blueprint with ghpc for ${exampleFile}" exit 1 @@ -50,6 +50,10 @@ run_test() { } for folder in ./*; do cd "$folder" + pkrdirs=() + while IFS= read -r -d $'\n'; do + pkrdirs+=("$REPLY") + done < <(find . -name "*.pkr.hcl" -printf '%h\n' | sort -u) if [ -f 'main.tf' ]; then tfpw=$(pwd) terraform init -no-color -backend=false >"${exampleFile}.init" || @@ -62,8 +66,16 @@ run_test() { echo "*** ERROR: terraform validate failed for ${example}, logs in ${tfpw}" exit 1 } + elif [ ${#pkrdirs[@]} -gt 0 ]; then + for pkrdir in "${pkrdirs[@]}"; do + packer validate -syntax-only "${pkrdir}" >/dev/null || + { + echo "*** ERROR: packer validate failed for ${example}" + exit 1 + } + done else - echo "terraform not found in folder ${BLUEPRINT}/${folder}. Skipping." + echo "neither packer nor terraform found in folder ${BLUEPRINT}/${folder}. Skipping." fi cd .. # back to blueprint folder done @@ -77,10 +89,27 @@ run_test() { } check_background() { - if ! wait -n; then - wait - echo "*** ERROR: a test failed. Exiting with status 1." - exit 1 + # "wait -n" was introduced in bash 4.3; support CentOS 7: 4.2 and MacOS: 3.2! + if [[ "${BASH_VERSINFO[0]}" -ge 5 || "${BASH_VERSINFO[0]}" -eq 4 && "${BASH_VERSINFO[1]}" -ge 3 ]]; then + if ! wait -n; then + wait + echo "*** ERROR: a test failed. Exiting with status 1." + exit 1 + + fi + else + failed=0 + for pid in "${pids[@]}"; do + if ! wait "$pid"; then + failed=1 + fi + done + pids=() + + if [[ $failed -eq 1 ]]; then + echo "*** ERROR: a test failed. Exiting with status 1." + exit 1 + fi fi } @@ -88,11 +117,13 @@ CONFIGS=$(find examples/ tools/validate_configs/test_configs/ -name "*.yaml" -ty cwd=$(pwd) NPROCS=${NPROCS:-$(nproc)} echo "Running tests in $NPROCS processes" +pids=() for example in $CONFIGS; do JNUM=$(jobs | wc -l) # echo "$JNUM jobs running" if [ "$JNUM" -lt "$NPROCS" ]; then run_test "$example" & + pids+=("$!") else # echo "Reached max number of parallel tests (${JNUM}). Waiting for one to finish." check_background