Run config validation test in a different directory than the repo, fix bugs found when making the change

Makefile

@@ -47,7 +47,7 @@ install: install-user
 
 endif
 
-tests: warn-terraform-version warn-packer-version test-engine validate_configs packer-check
+tests: ghpc install warn-terraform-version warn-packer-version test-engine validate_configs packer-check
 
 format: warn-go-version warn-terraform-version warn-packer-version terraform-format packer-format
 	$(info **************** formatting go code *******************)

pkg/config/config.go

@@ -32,7 +32,6 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"hpc-toolkit/pkg/modulereader"
-	"hpc-toolkit/pkg/sourcereader"
 )
 
 const (
@@ -402,8 +401,7 @@ func createModuleInfo(
 	modsInfo := make(map[string]modulereader.ModuleInfo)
 	for _, mod := range deploymentGroup.Modules {
 		if _, exists := modsInfo[mod.Source]; !exists {
-			reader := sourcereader.Factory(mod.Source)
-			ri, err := reader.GetModuleInfo(mod.Source, mod.Kind)
+			ri, err := modulereader.GetModuleInfo(mod.Source, mod.Kind)
 			if err != nil {
 				log.Fatalf(
 					"failed to get info for module at %s while setting dc.ModulesInfo: %e",

pkg/config/expand.go

@@ -23,7 +23,6 @@ import (
 	"path/filepath"
 
 	"hpc-toolkit/pkg/modulereader"
-	"hpc-toolkit/pkg/sourcereader"
 
 	"golang.org/x/exp/slices"
 )
@@ -471,8 +470,7 @@ func expandSimpleVariable(
 			context.varString)
 	}
 	refMod := refGrp.Modules[refModIndex]
-	reader := sourcereader.Factory(refMod.Source)
-	modInfo, err := reader.GetModuleInfo(refMod.Source, refMod.Kind)
+	modInfo, err := modulereader.GetModuleInfo(refMod.Source, refMod.Kind)
 	if err != nil {
 		log.Fatalf(
 			"failed to get info for module at %s while expanding variables: %e",

pkg/config/validate.go

@@ -23,7 +23,6 @@ import (
 	"strings"
 
 	"hpc-toolkit/pkg/modulereader"
-	"hpc-toolkit/pkg/sourcereader"
 	"hpc-toolkit/pkg/validators"
 
 	"github.com/pkg/errors"
@@ -263,8 +262,7 @@ func validateSettings(
 func (dc DeploymentConfig) validateModuleSettings() error {
 	for _, grp := range dc.Config.DeploymentGroups {
 		for _, mod := range grp.Modules {
-			reader := sourcereader.Factory(mod.Source)
-			info, err := reader.GetModuleInfo(mod.Source, mod.Kind)
+			info, err := modulereader.GetModuleInfo(mod.Source, mod.Kind)
 			if err != nil {
 				errStr := "failed to get info for module at %s while validating module settings"
 				return errors.Wrapf(err, errStr, mod.Source)

pkg/deploymentio/local.go

@@ -64,7 +64,7 @@ func (b *Local) CopyFromPath(src string, dst string) error {
 	return copy.Copy(absPath, dst)
 }
 
-// CopyFromFS copyes the embedded source file to the destination file
+// CopyFromFS copies the embedded source file to the destination file
 func (b *Local) CopyFromFS(fs BaseFS, src string, dst string) error {
 	data, err := fs.ReadFile(src)
 	if err != nil {

pkg/modulereader/hcl_utils.go

@@ -16,25 +16,20 @@ package modulereader
 
 import (
 	"fmt"
+	"hpc-toolkit/pkg/sourcereader"
 	"os"
-	"strings"
 
 	"github.com/hashicorp/terraform-config-inspect/tfconfig"
 )
 
-// isEmbeddedPath checks if a source path points to an embedded module
-func isEmbeddedPath(source string) bool {
-	return strings.HasPrefix(source, "modules/") || strings.HasPrefix(source, "community/modules/")
-}
-
 // getHCLInfo is wrapped by SourceReader interface which supports multiple
 // sources and stores remote modules locally, so the given source parameter to
 // getHCLInfo is only a local path.
 func getHCLInfo(source string) (ModuleInfo, error) {
 	var module *tfconfig.Module
 	ret := ModuleInfo{}
 
-	if isEmbeddedPath(source) {
+	if sourcereader.IsEmbeddedPath(source) {
 		if !tfconfig.IsModuleDirOnFilesystem(tfconfig.WrapFS(ModuleFS), source) {
 			return ret, fmt.Errorf("Source is not a terraform or packer module: %s", source)
 		}

pkg/modulereader/packerreader.go

@@ -18,10 +18,11 @@ package modulereader
 
 import (
 	"fmt"
-	"io"
+	"hpc-toolkit/pkg/sourcereader"
 	"io/ioutil"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
 )
 
@@ -39,15 +40,15 @@ func addTfExtension(filename string) {
 	newFilename := fmt.Sprintf("%s.tf", filename)
 	if err := os.Rename(filename, newFilename); err != nil {
 		log.Fatalf(
-			"failed to add .tf extension to %s needed to get info on packer module: %e",
+			"failed to add .tf extension to %s needed to get info on packer module: %v",
 			filename, err)
 	}
 }
 
 func getHCLFiles(dir string) []string {
 	allFiles, err := ioutil.ReadDir(dir)
 	if err != nil {
-		log.Fatalf("Failed to read packer source directory %s", dir)
+		log.Fatalf("Failed to read packer source directory at %s: %v", dir, err)
 	}
 	var hclFiles []string
 	for _, f := range allFiles {
@@ -61,60 +62,32 @@ func getHCLFiles(dir string) []string {
 	return hclFiles
 }
 
-func copyHCLFilesToTmp(dir string) (string, []string, error) {
-	tmpDir, err := ioutil.TempDir("", "pkwriter-*")
-	if err != nil {
-		return "", []string{}, fmt.Errorf(
-			"failed to create temp directory for packer reader")
-	}
-	hclFiles := getHCLFiles(dir)
-	var hclFilePaths []string
-
-	for _, hclFilename := range hclFiles {
-
-		// Open file for copying
-		hclFile, err := os.Open(hclFilename)
-		if err != nil {
-			return "", hclFiles, fmt.Errorf(
-				"failed to open packer HCL file %s: %v", hclFilename, err)
-		}
-		defer hclFile.Close()
-
-		// Create a file to copy to
-		destPath := filepath.Join(tmpDir, filepath.Base(hclFilename))
-		destination, err := os.Create(destPath)
-		if err != nil {
-			return "", hclFiles, fmt.Errorf(
-				"failed to create copy of packer HCL file %s: %v", hclFilename, err)
-		}
-		defer destination.Close()
-
-		// Copy
-		if _, err := io.Copy(destination, hclFile); err != nil {
-			return "", hclFiles, fmt.Errorf(
-				"failed to copy packer module at %s to temporary directory to inspect: %v",
-				dir, err)
-		}
-		hclFilePaths = append(hclFilePaths, destPath)
-	}
-	return tmpDir, hclFilePaths, nil
-}
-
 // GetInfo reads the ModuleInfo for a packer module
 func (r PackerReader) GetInfo(source string) (ModuleInfo, error) {
 	if modInfo, ok := r.allModInfo[source]; ok {
 		return modInfo, nil
 	}
-	tmpDir, packerFiles, err := copyHCLFilesToTmp(source)
+
+	tmpDir, err := ioutil.TempDir("", "pkwriter-*")
 	if err != nil {
-		return ModuleInfo{}, err
+		return ModuleInfo{}, fmt.Errorf(
+			"failed to create temp directory for packer reader")
 	}
 	defer os.RemoveAll(tmpDir)
 
+	modName := path.Base(source)
+	modPath := path.Join(tmpDir, modName)
+
+	sourceReader := sourcereader.Factory(source)
+	if err = sourceReader.GetModule(source, modPath); err != nil {
+		return ModuleInfo{}, err
+	}
+	packerFiles := getHCLFiles(modPath)
+
 	for _, packerFile := range packerFiles {
 		addTfExtension(packerFile)
 	}
-	modInfo, err := getHCLInfo(tmpDir)
+	modInfo, err := getHCLInfo(modPath)
 	if err != nil {
 		return modInfo, fmt.Errorf("PackerReader: %v", err)
 	}