Merge pull request #827 from heyealex/tests/run-outside-repo

Run config validation test in a different directory than the repo, fix bugs found when making the change

pkg/config/config.go

@@ -32,7 +32,6 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"hpc-toolkit/pkg/modulereader"
-	"hpc-toolkit/pkg/sourcereader"
 )
 
 const (
@@ -398,8 +397,7 @@ func createModuleInfo(
 	modsInfo := make(map[string]modulereader.ModuleInfo)
 	for _, mod := range deploymentGroup.Modules {
 		if _, exists := modsInfo[mod.Source]; !exists {
-			reader := sourcereader.Factory(mod.Source)
-			ri, err := reader.GetModuleInfo(mod.Source, mod.Kind)
+			ri, err := modulereader.GetModuleInfo(mod.Source, mod.Kind)
 			if err != nil {
 				log.Fatalf(
 					"failed to get info for module at %s while setting dc.ModulesInfo: %e",

pkg/config/expand.go

@@ -23,7 +23,6 @@ import (
 	"path/filepath"
 
 	"hpc-toolkit/pkg/modulereader"
-	"hpc-toolkit/pkg/sourcereader"
 
 	"golang.org/x/exp/slices"
 )
@@ -526,8 +525,7 @@ func expandSimpleVariable(
 			context.varString)
 	}
 	refMod := refGrp.Modules[refModIndex]
-	reader := sourcereader.Factory(refMod.Source)
-	modInfo, err := reader.GetModuleInfo(refMod.Source, refMod.Kind)
+	modInfo, err := modulereader.GetModuleInfo(refMod.Source, refMod.Kind)
 	if err != nil {
 		log.Fatalf(
 			"failed to get info for module at %s while expanding variables: %e",

pkg/config/validate.go

@@ -23,7 +23,6 @@ import (
 	"strings"
 
 	"hpc-toolkit/pkg/modulereader"
-	"hpc-toolkit/pkg/sourcereader"
 	"hpc-toolkit/pkg/validators"
 
 	"github.com/pkg/errors"
@@ -263,8 +262,7 @@ func validateSettings(
 func (dc DeploymentConfig) validateModuleSettings() error {
 	for _, grp := range dc.Config.DeploymentGroups {
 		for _, mod := range grp.Modules {
-			reader := sourcereader.Factory(mod.Source)
-			info, err := reader.GetModuleInfo(mod.Source, mod.Kind)
+			info, err := modulereader.GetModuleInfo(mod.Source, mod.Kind)
 			if err != nil {
 				errStr := "failed to get info for module at %s while validating module settings"
 				return errors.Wrapf(err, errStr, mod.Source)

pkg/deploymentio/local.go

@@ -64,7 +64,7 @@ func (b *Local) CopyFromPath(src string, dst string) error {
 	return copy.Copy(absPath, dst)
 }
 
-// CopyFromFS copyes the embedded source file to the destination file
+// CopyFromFS copies the embedded source file to the destination file
 func (b *Local) CopyFromFS(fs BaseFS, src string, dst string) error {
 	data, err := fs.ReadFile(src)
 	if err != nil {

pkg/modulereader/hcl_utils.go

@@ -16,25 +16,20 @@ package modulereader
 
 import (
 	"fmt"
+	"hpc-toolkit/pkg/sourcereader"
 	"os"
-	"strings"
 
 	"github.com/hashicorp/terraform-config-inspect/tfconfig"
 )
 
-// isEmbeddedPath checks if a source path points to an embedded module
-func isEmbeddedPath(source string) bool {
-	return strings.HasPrefix(source, "modules/") || strings.HasPrefix(source, "community/modules/")
-}
-
 // getHCLInfo is wrapped by SourceReader interface which supports multiple
 // sources and stores remote modules locally, so the given source parameter to
 // getHCLInfo is only a local path.
 func getHCLInfo(source string) (ModuleInfo, error) {
 	var module *tfconfig.Module
 	ret := ModuleInfo{}
 
-	if isEmbeddedPath(source) {
+	if sourcereader.IsEmbeddedPath(source) {
 		if !tfconfig.IsModuleDirOnFilesystem(tfconfig.WrapFS(ModuleFS), source) {
 			return ret, fmt.Errorf("Source is not a terraform or packer module: %s", source)
 		}

pkg/modulereader/packerreader.go

@@ -18,10 +18,11 @@ package modulereader
 
 import (
 	"fmt"
-	"io"
+	"hpc-toolkit/pkg/sourcereader"
 	"io/ioutil"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
 )
 
@@ -39,15 +40,15 @@ func addTfExtension(filename string) {
 	newFilename := fmt.Sprintf("%s.tf", filename)
 	if err := os.Rename(filename, newFilename); err != nil {
 		log.Fatalf(
-			"failed to add .tf extension to %s needed to get info on packer module: %e",
+			"failed to add .tf extension to %s needed to get info on packer module: %v",
 			filename, err)
 	}
 }
 
 func getHCLFiles(dir string) []string {
 	allFiles, err := ioutil.ReadDir(dir)
 	if err != nil {
-		log.Fatalf("Failed to read packer source directory %s", dir)
+		log.Fatalf("Failed to read packer source directory at %s: %v", dir, err)
 	}
 	var hclFiles []string
 	for _, f := range allFiles {
@@ -61,60 +62,32 @@ func getHCLFiles(dir string) []string {
 	return hclFiles
 }
 
-func copyHCLFilesToTmp(dir string) (string, []string, error) {
-	tmpDir, err := ioutil.TempDir("", "pkwriter-*")
-	if err != nil {
-		return "", []string{}, fmt.Errorf(
-			"failed to create temp directory for packer reader")
-	}
-	hclFiles := getHCLFiles(dir)
-	var hclFilePaths []string
-
-	for _, hclFilename := range hclFiles {
-
-		// Open file for copying
-		hclFile, err := os.Open(hclFilename)
-		if err != nil {
-			return "", hclFiles, fmt.Errorf(
-				"failed to open packer HCL file %s: %v", hclFilename, err)
-		}
-		defer hclFile.Close()
-
-		// Create a file to copy to
-		destPath := filepath.Join(tmpDir, filepath.Base(hclFilename))
-		destination, err := os.Create(destPath)
-		if err != nil {
-			return "", hclFiles, fmt.Errorf(
-				"failed to create copy of packer HCL file %s: %v", hclFilename, err)
-		}
-		defer destination.Close()
-
-		// Copy
-		if _, err := io.Copy(destination, hclFile); err != nil {
-			return "", hclFiles, fmt.Errorf(
-				"failed to copy packer module at %s to temporary directory to inspect: %v",
-				dir, err)
-		}
-		hclFilePaths = append(hclFilePaths, destPath)
-	}
-	return tmpDir, hclFilePaths, nil
-}
-
 // GetInfo reads the ModuleInfo for a packer module
 func (r PackerReader) GetInfo(source string) (ModuleInfo, error) {
 	if modInfo, ok := r.allModInfo[source]; ok {
 		return modInfo, nil
 	}
-	tmpDir, packerFiles, err := copyHCLFilesToTmp(source)
+
+	tmpDir, err := ioutil.TempDir("", "pkwriter-*")
 	if err != nil {
-		return ModuleInfo{}, err
+		return ModuleInfo{}, fmt.Errorf(
+			"failed to create temp directory for packer reader")
 	}
 	defer os.RemoveAll(tmpDir)
 
+	modName := path.Base(source)
+	modPath := path.Join(tmpDir, modName)
+
+	sourceReader := sourcereader.Factory(source)
+	if err = sourceReader.GetModule(source, modPath); err != nil {
+		return ModuleInfo{}, err
+	}
+	packerFiles := getHCLFiles(modPath)
+
 	for _, packerFile := range packerFiles {
 		addTfExtension(packerFile)
 	}
-	modInfo, err := getHCLInfo(tmpDir)
+	modInfo, err := getHCLInfo(modPath)
 	if err != nil {
 		return modInfo, fmt.Errorf("PackerReader: %v", err)
 	}

pkg/modulereader/resreader.go

@@ -19,7 +19,12 @@ package modulereader
 
 import (
 	"embed"
+	"fmt"
+	"hpc-toolkit/pkg/sourcereader"
+	"io/ioutil"
 	"log"
+	"path"
+	"strings"
 )
 
 // ModuleFS contains embedded modules (./modules) for use in building
@@ -50,6 +55,47 @@ func (i ModuleInfo) GetOutputsAsMap() map[string]VarInfo {
 	return outputsMap
 }
 
+// GetModuleInfo gathers information about a module at a given source using the
+// tfconfig package. For applicable sources, this function also stages the
+// module contents in a local temp directory and will add required APIs to be
+// enabled for that module.
+func GetModuleInfo(source string, kind string) (ModuleInfo, error) {
+	var modPath string
+	switch {
+	case sourcereader.IsGitPath(source):
+		tmpDir, err := ioutil.TempDir("", "module-*")
+		if err != nil {
+			return ModuleInfo{}, err
+		}
+		modPath = path.Join(tmpDir, "module")
+		sourceReader := sourcereader.Factory(source)
+		if err = sourceReader.GetModule(source, modPath); err != nil {
+			return ModuleInfo{}, fmt.Errorf("failed to clone git module at %s: %v", source, err)
+		}
+
+	case sourcereader.IsEmbeddedPath(source) || sourcereader.IsLocalPath(source):
+		modPath = source
+
+	default:
+		return ModuleInfo{}, fmt.Errorf("Source is not valid: %s", source)
+	}
+
+	reader := Factory(kind)
+	mi, err := reader.GetInfo(modPath)
+
+	// add APIs required by the module, if known
+	if sourcereader.IsEmbeddedPath(source) {
+		mi.RequiredApis = defaultAPIList(modPath)
+	} else if sourcereader.IsLocalPath(source) {
+		if idx := strings.Index(modPath, "/community/modules/"); idx != -1 {
+			mi.RequiredApis = defaultAPIList(modPath[idx+1:])
+		} else if idx := strings.Index(modPath, "/modules/"); idx != -1 {
+			mi.RequiredApis = defaultAPIList(modPath[idx+1:])
+		}
+	}
+	return mi, err
+}
+
 // ModReader is a module reader interface
 type ModReader interface {
 	GetInfo(path string) (ModuleInfo, error)
@@ -81,3 +127,135 @@ func Factory(kind string) ModReader {
 	log.Fatalf("Invalid request to create a reader of kind %s", kind)
 	return nil
 }
+
+func defaultAPIList(source string) []string {
+	// API lists at
+	// https://console.cloud.google.com/apis/dashboard and
+	// https://console.cloud.google.com/apis/library
+	staticAPIMap := map[string][]string{
+		"community/modules/compute/SchedMD-slurm-on-gcp-partition": {
+			"compute.googleapis.com",
+		},
+		"community/modules/compute/htcondor-execute-point": {
+			"compute.googleapis.com",
+		},
+		"community/modules/compute/pbspro-execution": {
+			"compute.googleapis.com",
+			"storage.googleapis.com",
+		},
+		"community/modules/compute/schedmd-slurm-gcp-v5-partition": {
+			"compute.googleapis.com",
+		},
+		"community/modules/database/slurm-cloudsql-federation": {
+			"bigqueryconnection.googleapis.com",
+			"sqladmin.googleapis.com",
+		},
+		"community/modules/file-system/DDN-EXAScaler": {
+			"compute.googleapis.com",
+			"deploymentmanager.googleapis.com",
+			"iam.googleapis.com",
+			"runtimeconfig.googleapis.com",
+		},
+		"community/modules/file-system/Intel-DAOS": {
+			"compute.googleapis.com",
+			"iam.googleapis.com",
+			"secretmanager.googleapis.com",
+		},
+		"community/modules/file-system/nfs-server": {
+			"compute.googleapis.com",
+		},
+		"community/modules/project/new-project": {
+			"admin.googleapis.com",
+			"cloudresourcemanager.googleapis.com",
+			"cloudbilling.googleapis.com",
+			"iam.googleapis.com",
+		},
+		"community/modules/project/service-account": {
+			"iam.googleapis.com",
+		},
+		"community/modules/project/service-enablement": {
+			"serviceusage.googleapis.com",
+		},
+		"community/modules/scheduler/SchedMD-slurm-on-gcp-controller": {
+			"compute.googleapis.com",
+		},
+		"community/modules/scheduler/SchedMD-slurm-on-gcp-login-node": {
+			"compute.googleapis.com",
+		},
+		"modules/scheduler/batch-job-template": {
+			"batch.googleapis.com",
+			"compute.googleapis.com",
+		},
+		"modules/scheduler/batch-login-node": {
+			"batch.googleapis.com",
+			"compute.googleapis.com",
+			"storage.googleapis.com",
+		},
+		"community/modules/scheduler/htcondor-configure": {
+			"iam.googleapis.com",
+			"secretmanager.googleapis.com",
+		},
+		"community/modules/scheduler/pbspro-client": {
+			"compute.googleapis.com",
+			"storage.googleapis.com",
+		},
+		"community/modules/scheduler/pbspro-server": {
+			"compute.googleapis.com",
+			"storage.googleapis.com",
+		},
+		"community/modules/scheduler/schedmd-slurm-gcp-v5-controller": {
+			"compute.googleapis.com",
+			"iam.googleapis.com",
+			"pubsub.googleapis.com",
+			"secretmanager.googleapis.com",
+		},
+		"community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid": {
+			"compute.googleapis.com",
+			"pubsub.googleapis.com",
+		},
+		"community/modules/scheduler/schedmd-slurm-gcp-v5-login": {
+			"compute.googleapis.com",
+		},
+		"community/modules/scripts/htcondor-install": {},
+		"community/modules/scripts/omnia-install":    {},
+		"community/modules/scripts/pbspro-preinstall": {
+			"iam.googleapis.com",
+			"storage.googleapis.com",
+		},
+		"community/modules/scripts/pbspro-install": {},
+		"community/modules/scripts/pbspro-qmgr":    {},
+		"community/modules/scripts/spack-install":  {},
+		"community/modules/scripts/wait-for-startup": {
+			"compute.googleapis.com",
+		},
+		"modules/compute/vm-instance": {
+			"compute.googleapis.com",
+		},
+		"modules/file-system/filestore": {
+			"file.googleapis.com",
+		},
+		"modules/file-system/pre-existing-network-storage": {},
+		"modules/monitoring/dashboard": {
+			"stackdriver.googleapis.com",
+		},
+		"modules/network/pre-existing-vpc": {
+			"compute.googleapis.com",
+		},
+		"modules/network/vpc": {
+			"compute.googleapis.com",
+		},
+		"modules/packer/custom-image": {
+			"compute.googleapis.com",
+			"storage.googleapis.com",
+		},
+		"modules/scripts/startup-script": {
+			"storage.googleapis.com",
+		},
+	}
+
+	requiredAPIs, found := staticAPIMap[source]
+	if !found {
+		return []string{}
+	}
+	return requiredAPIs
+}