feat: add provider version to metadata generation (#2496)

Co-authored-by: Zheng Qin <zhengqin@google.com>
Co-authored-by: Andrew Peabody <andrewpeabody@google.com>
Co-authored-by: Bharath KKB <bharathkrishnakb@gmail.com>

cli/bpmetadata/cmd.go

@@ -249,7 +249,8 @@ func CreateBlueprintMetadata(bpPath string, bpMetadataObj *BlueprintMetadata) (*
 	// get blueprint requirements
 	rolesCfgPath := path.Join(repoDetails.Source.BlueprintRootPath, tfRolesFileName)
 	svcsCfgPath := path.Join(repoDetails.Source.BlueprintRootPath, tfServicesFileName)
-	requirements, err := getBlueprintRequirements(rolesCfgPath, svcsCfgPath)
+	versionsCfgPath := path.Join(repoDetails.Source.BlueprintRootPath, tfVersionsFileName)
+	requirements, err := getBlueprintRequirements(rolesCfgPath, svcsCfgPath, versionsCfgPath)
 	if err != nil {
 		Log.Info("skipping blueprint requirements since roles and/or services configurations were not found as per https://tinyurl.com/tf-iam and https://tinyurl.com/tf-services")
 	} else {

cli/bpmetadata/int-test/goldens/golden-metadata.yaml

@@ -249,3 +249,8 @@ spec:
       - cloudresourcemanager.googleapis.com
       - compute.googleapis.com
       - serviceusage.googleapis.com
+    providerVersions:
+      - source: hashicorp/google
+        version: ">= 4.42, < 5.0"
+      - source: hashicorp/random
+        version: ">= 2.1"

cli/bpmetadata/proto/bpmetadata.proto

@@ -195,7 +195,7 @@ message BlueprintRequirements {
 
   // Required provider versions.
   // Gen: auto-generated from required providers block.
-  repeated ProviderVersion provider_versions = 3; // @gotags: json:"versions,omitempty" yaml:"providerVersions,omitempty"
+  repeated ProviderVersion provider_versions = 3; // @gotags: json:"providerVersions,omitempty" yaml:"providerVersions,omitempty"
 }
 
 // ProviderVersion defines the required version for a provider.

cli/bpmetadata/schema/gcp-blueprint-metadata.json

@@ -440,7 +440,7 @@
           },
           "type": "array"
         },
-        "versions": {
+        "providerVersions": {
           "items": {
             "$ref": "#/$defs/ProviderVersion"
           },

cli/bpmetadata/tfconfig.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strings"
 
 	hcl "github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/gohcl"
@@ -188,6 +189,36 @@ func parseBlueprintVersion(versionsFile *hcl.File, diags hcl.Diagnostics) (strin
 	return "", nil
 }
 
+// parseBlueprintProviderVersions gets the blueprint provider_versions from the provided config
+// from the required_providers block.
+func parseBlueprintProviderVersions(versionsFile *hcl.File) ([]*ProviderVersion, error) {
+	var v []*ProviderVersion
+	// parse out the required providers from the config
+	var hclModule tfconfig.Module
+	hclModule.RequiredProviders = make(map[string]*tfconfig.ProviderRequirement)
+	diags := tfconfig.LoadModuleFromFile(versionsFile, &hclModule)
+	err := hasHclErrors(diags)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, providerData := range hclModule.RequiredProviders {
+		if providerData.Source == "" {
+			Log.Info("Not found source in provider settings\n")
+			continue
+		}
+		if len(providerData.VersionConstraints) == 0 {
+			Log.Info("Not found version in provider settings\n")
+			continue
+		}
+		v = append(v, &ProviderVersion{
+			Source:  providerData.Source,
+			Version: strings.Join(providerData.VersionConstraints, ", "),
+		})
+	}
+	return v, nil
+}
+
 // getBlueprintInterfaces gets the variables and outputs associated
 // with the blueprint
 func getBlueprintInterfaces(configPath string) (*BlueprintInterface, error) {
@@ -253,7 +284,7 @@ func getBlueprintOutput(modOut *tfconfig.Output) *BlueprintOutput {
 
 // getBlueprintRequirements gets the services and roles associated
 // with the blueprint
-func getBlueprintRequirements(rolesConfigPath, servicesConfigPath string) (*BlueprintRequirements, error) {
+func getBlueprintRequirements(rolesConfigPath, servicesConfigPath, versionsConfigPath string) (*BlueprintRequirements, error) {
 	//parse blueprint roles
 	p := hclparse.NewParser()
 	rolesFile, diags := p.ParseHCLFile(rolesConfigPath)
@@ -279,10 +310,33 @@ func getBlueprintRequirements(rolesConfigPath, servicesConfigPath string) (*Blue
 		return nil, err
 	}
 
+	versionCfgFileExists, _ := fileExists(versionsConfigPath)
+
+	if !versionCfgFileExists {
+		return &BlueprintRequirements{
+			Roles:    r,
+			Services: s,
+		}, nil
+	}
+
+	//parse blueprint provider versions
+	versionsFile, diags := p.ParseHCLFile(versionsConfigPath)
+	err = hasHclErrors(diags)
+	if err != nil {
+		return nil, err
+	}
+
+	v, err := parseBlueprintProviderVersions(versionsFile)
+	if err != nil {
+		return nil, err
+	}
+
 	return &BlueprintRequirements{
-		Roles:    r,
-		Services: s,
+		Roles:            r,
+		Services:         s,
+		ProviderVersions: v,
 	}, nil
+
 }
 
 // parseBlueprintRoles gets the roles required for the blueprint to be provisioned
@@ -399,15 +453,15 @@ func hasTfconfigErrors(diags tfconfig.Diagnostics) error {
 // MergeExistingConnections merges existing connections from an old BlueprintInterface into a new one,
 // preserving manually authored connections.
 func mergeExistingConnections(newInterfaces, existingInterfaces *BlueprintInterface) {
-    if existingInterfaces == nil {
-        return // Nothing to merge if existingInterfaces is nil
-    }
-
-    for i, variable := range newInterfaces.Variables {
-        for _, existingVariable := range existingInterfaces.Variables {
-            if variable.Name == existingVariable.Name && existingVariable.Connections != nil {
-                newInterfaces.Variables[i].Connections = existingVariable.Connections
-            }
-        }
-    }
+	if existingInterfaces == nil {
+		return // Nothing to merge if existingInterfaces is nil
+	}
+
+	for i, variable := range newInterfaces.Variables {
+		for _, existingVariable := range existingInterfaces.Variables {
+			if variable.Name == existingVariable.Name && existingVariable.Connections != nil {
+				newInterfaces.Variables[i].Connections = existingVariable.Connections
+			}
+		}
+	}
 }

cli/bpmetadata/tfconfig_test.go

@@ -11,9 +11,9 @@ import (
 )
 
 const (
-	tfTestdataPath = "../testdata/bpmetadata/tf"
+	tfTestdataPath       = "../testdata/bpmetadata/tf"
 	metadataTestdataPath = "../testdata/bpmetadata/metadata"
-	interfaces     = "sample-module"
+	interfaces           = "sample-module"
 )
 
 func TestTFInterfaces(t *testing.T) {
@@ -268,25 +268,58 @@ func TestTFRoles(t *testing.T) {
 	}
 }
 
+func TestTFProviderVersions(t *testing.T) {
+	tests := []struct {
+		name                 string
+		configName           string
+		wantProviderVersions []*ProviderVersion
+	}{
+		{
+			name:       "Simple list of provider versions",
+			configName: "versions-beta.tf",
+			wantProviderVersions: []*ProviderVersion{
+				{
+					Source:  "hashicorp/google",
+					Version: ">= 4.4.0, < 7",
+				},
+				{
+					Source:  "hashicorp/google-beta",
+					Version: ">= 4.4.0, < 7",
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := hclparse.NewParser()
+			content, _ := p.ParseHCLFile(path.Join(tfTestdataPath, tt.configName))
+			got, err := parseBlueprintProviderVersions(content)
+			require.NoError(t, err)
+			assert.Equal(t, got, tt.wantProviderVersions)
+		})
+	}
+}
+
 func TestMergeExistingConnections(t *testing.T) {
 	tests := []struct {
-		name                string
-		newInterfacesFile   string
+		name                   string
+		newInterfacesFile      string
 		existingInterfacesFile string
 	}{
 		{
-			name:                "No existing connections",
-			newInterfacesFile:   "new_interfaces_no_connections_metadata.yaml",
+			name:                   "No existing connections",
+			newInterfacesFile:      "new_interfaces_no_connections_metadata.yaml",
 			existingInterfacesFile: "existing_interfaces_without_connections_metadata.yaml",
 		},
 		{
-			name:                "One existing connection is preserved",
-			newInterfacesFile:   "new_interfaces_no_connections_metadata.yaml",
+			name:                   "One existing connection is preserved",
+			newInterfacesFile:      "new_interfaces_no_connections_metadata.yaml",
 			existingInterfacesFile: "existing_interfaces_with_one_connection_metadata.yaml",
 		},
 		{
-			name:                "Multiple existing connections are preserved",
-			newInterfacesFile:   "new_interfaces_no_connections_metadata.yaml",
+			name:                   "Multiple existing connections are preserved",
+			newInterfacesFile:      "new_interfaces_no_connections_metadata.yaml",
 			existingInterfacesFile: "existing_interfaces_with_some_connections_metadata.yaml",
 		},
 	}
@@ -309,3 +342,29 @@ func TestMergeExistingConnections(t *testing.T) {
 		})
 	}
 }
+
+func TestTFIncompleteProviderVersions(t *testing.T) {
+	tests := []struct {
+		name       string
+		configName string
+	}{
+		{
+			name:       "Empty list of provider versions",
+			configName: "provider-versions-empty.tf",
+		},
+		{
+			name:       "Missing ProviderVersion field",
+			configName: "provider-versions-bad.tf",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := hclparse.NewParser()
+			content, _ := p.ParseHCLFile(path.Join(tfTestdataPath, tt.configName))
+			got, err := parseBlueprintProviderVersions(content)
+			require.NoError(t, err)
+			assert.Nil(t, got)
+		})
+	}
+}

cli/testdata/bpmetadata/tf/provider-versions-bad.tf

@@ -0,0 +1,19 @@
+terraform {
+  required_version = ">= 0.13.0"
+
+  required_providers {
+    google = {
+      version = ">= 4.4.0, < 6"
+    }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+    }
+  }
+
+  provider_meta "google" {
+    module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v23.1.0"
+  }
+  provider_meta "google-beta" {
+    module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v23.1.0"
+  }
+}

cli/testdata/bpmetadata/tf/provider-versions-empty.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_version = ">= 0.13.0"
+
+  required_providers {
+  }
+
+  provider_meta "google" {
+    module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v23.1.0"
+  }
+  provider_meta "google-beta" {
+    module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v23.1.0"
+  }
+}