Support GITHUB_TOKEN for HTTP Requests to github.com

cmd/terraform.go

@@ -1,6 +1,10 @@
 package cmd
 
 import (
+	"os"
+	"path/filepath"
+	"strings"
+
 	"github.com/samber/lo"
 	"github.com/spf13/cobra"
 
@@ -42,6 +46,29 @@ var terraformCmd = &cobra.Command{
 		// Check Atmos configuration
 		checkAtmosConfig()
 
+		//Load stack from Github
+		folderFlag, _ := cmd.Flags().GetString("folder")
+		if folderFlag != "" && u.IsGithubURL(info.Stack) {
+
+			data, err := u.DownloadFileFromGitHub(info.Stack)
+			if err != nil {
+				u.LogErrorAndExit(schema.AtmosConfiguration{}, err)
+			}
+			fileName := u.ParseFilenameFromURL(info.Stack)
+			if fileName == "" {
+				fileName = "stack.yaml" // fallback
+			}
+			localPath := filepath.Join(folderFlag, fileName)
+
+			// Overwrite if it exists
+			err = os.WriteFile(localPath, data, 0o644)
+			if err != nil {
+				u.LogErrorAndExit(schema.AtmosConfiguration{}, err)
+			}
+			shortStackName := strings.TrimSuffix(fileName, filepath.Ext(fileName))
+			info.Stack = shortStackName
+		}
+
 		err = e.ExecuteTerraform(info)
 		if err != nil {
 			u.LogErrorAndExit(schema.AtmosConfiguration{}, err)
@@ -53,5 +80,10 @@ func init() {
 	// https://github.com/spf13/cobra/issues/739
 	terraformCmd.DisableFlagParsing = true
 	terraformCmd.PersistentFlags().StringP("stack", "s", "", "atmos terraform <terraform_command> <component> -s <stack>")
+	terraformCmd.PersistentFlags().String(
+		"folder",
+		"",
+		"If set, download the remote stack file into this folder, then treat it as a local stack",
+	)
 	RootCmd.AddCommand(terraformCmd)
 }

internal/exec/stack_processor_utils.go

@@ -1929,6 +1929,23 @@ func GetFileContent(filePath string) (string, error) {
 		return fmt.Sprintf("%s", existingContent), nil
 	}
 
+	//Check if its Github remote URL to single file
+	// Shoudl be decommised as the direct hook into map does not work
+	/*parsedURL, err := url.Parse(filePath) // Parse the URL
+	if err != nil {
+		u.LogInfo(schema.AtmosConfiguration{}, fmt.Sprintf("Filepath is local: %s", filePath))
+	} else {
+		if parsedURL.Host == "github.com" && parsedURL.Scheme == "https" {
+			u.LogDebug(schema.AtmosConfiguration{}, fmt.Sprintf("Fetching GitHub source: %s", filePath))
+			fileContents, err := u.DownloadFileFromGitHub(filePath)
+			if err != nil {
+				return "", fmt.Errorf("failed to download GitHub file: %w", err)
+			}
+			getFileContentSyncMap.Store(filePath, fileContents)
+			return string(fileContents), nil
+		}
+	}
+	*/
 	content, err := os.ReadFile(filePath)
 	if err != nil {
 		return "", err

internal/exec/utils.go

@@ -21,6 +21,7 @@ var (
 	commonFlags = []string{
 		"--stack",
 		"-s",
+		"--folder",
 		cfg.DryRunFlag,
 		cfg.SkipInitFlag,
 		cfg.KubeConfigConfigFlag,

internal/exec/vendor_utils.go

@@ -268,6 +268,8 @@ func ExecuteAtmosVendorInternal(
 		return fmt.Errorf("either 'spec.sources' or 'spec.imports' (or both) must be defined in the vendor config file '%s'", vendorConfigFileName)
 	}
 
+	var tempDir string
+
 	// Process imports and return all sources from all the imports and from `vendor.yaml`
 	sources, _, err := processVendorImports(
 		atmosConfig,
@@ -317,6 +319,16 @@ func ExecuteAtmosVendorInternal(
 		)
 	}
 
+	tempDir, err = os.MkdirTemp("", "atmos_vendor_")
+	if err != nil {
+		return fmt.Errorf("failed to create temp directory: %w", err)
+	}
+	defer func() {
+		if err := os.RemoveAll(tempDir); err != nil {
+			u.LogWarning(atmosConfig, fmt.Sprintf("failed to clean up temp directory %s: %v", tempDir, err))
+		}
+	}()
+
 	// Allow having duplicate targets in different sources.
 	// This can be used to vendor mixins (from local and remote sources) and write them to the same targets.
 	// TODO: consider adding a flag to `atmos vendor pull` to specify if duplicate targets are allowed or not.
@@ -359,7 +371,28 @@ func ExecuteAtmosVendorInternal(
 			return err
 		}
 
-		useOciScheme, useLocalFileSystem, sourceIsLocalFile := determineSourceType(&uri, vendorConfigFilePath)
+		useOciScheme, useLocalFileSystem, sourceIsLocalFile, isGitHubSource := determineSourceType(&uri, vendorConfigFilePath)
+
+		// Handle GitHub source
+		if isGitHubSource {
+			if dryRun {
+				u.LogInfo(atmosConfig, fmt.Sprintf("Dry run: Fetching GitHub source: %s", uri))
+			} else {
+				u.LogDebug(atmosConfig, fmt.Sprintf("Fetching GitHub source: %s", uri))
+				fileContents, err := u.DownloadFileFromGitHub(uri)
+				if err != nil {
+					return fmt.Errorf("failed to download GitHub file: %w", err)
+				}
+				// Save the downloaded file to the existing tempDir
+				tempGitHubFile := filepath.Join(tempDir, filepath.Base(uri))
+				err = os.WriteFile(tempGitHubFile, fileContents, os.ModePerm)
+				if err != nil {
+					return fmt.Errorf("failed to save GitHub file to temp location: %w", err)
+				}
+				// Update the URI to point to the saved file in the temp directory
+				uri = tempGitHubFile
+			}
+		}
 
 		// Determine package type
 		var pType pkgType
@@ -497,23 +530,35 @@ func shouldSkipSource(s *schema.AtmosVendorSource, component string, tags []stri
 	return (component != "" && s.Component != component) || (len(tags) > 0 && len(lo.Intersect(tags, s.Tags)) == 0)
 }
 
-func determineSourceType(uri *string, vendorConfigFilePath string) (bool, bool, bool) {
-	// Determine if the URI is an OCI scheme, a local file, or remote
+func determineSourceType(uri *string, vendorConfigFilePath string) (bool, bool, bool, bool) {
+	// Determine if the URI is an OCI scheme, a local file, a remote GitHub source, or a generic remote
 	useOciScheme := strings.HasPrefix(*uri, "oci://")
 	if useOciScheme {
 		*uri = strings.TrimPrefix(*uri, "oci://")
 	}
 
 	useLocalFileSystem := false
 	sourceIsLocalFile := false
+	isGitHubSource := false
+
+	// If not OCI, we proceed with checks
 	if !useOciScheme {
-		if absPath, err := u.JoinAbsolutePathWithPath(vendorConfigFilePath, *uri); err == nil {
-			uri = &absPath
-			useLocalFileSystem = true
-			sourceIsLocalFile = u.FileExists(*uri)
+		parsedURL, err := url.Parse(*uri)
+		if err != nil || parsedURL.Scheme == "" || parsedURL.Host == "" {
+			// Not a valid URL or no host: consider local filesystem
+			if absPath, err := u.JoinAbsolutePathWithPath(vendorConfigFilePath, *uri); err == nil {
+				uri = &absPath
+				useLocalFileSystem = true
+				sourceIsLocalFile = u.FileExists(*uri)
+			}
+		} else {
+			if parsedURL.Host == "github.com" && parsedURL.Scheme == "https" {
+				isGitHubSource = true
+			}
 		}
 	}
-	return useOciScheme, useLocalFileSystem, sourceIsLocalFile
+
+	return useOciScheme, useLocalFileSystem, sourceIsLocalFile, isGitHubSource
 }
 
 // sanitizeFileName replaces invalid characters and query strings with underscores for Windows.

pkg/config/config.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
@@ -15,6 +16,7 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/cloudposse/atmos/pkg/schema"
+	"github.com/cloudposse/atmos/pkg/utils"
 	u "github.com/cloudposse/atmos/pkg/utils"
 	"github.com/cloudposse/atmos/pkg/version"
 )
@@ -100,6 +102,8 @@ func InitCliConfig(configAndStacksInfo schema.ConfigAndStacksInfo, processStacks
 	// system dir (`/usr/local/etc/atmos` on Linux, `%LOCALAPPDATA%/atmos` on Windows)
 	// home dir (~/.atmos)
 	// current directory
+	// ENV var ATMOS_CLI_CONFIG_PATH
+	// ENV var ATMOS_REMOTE_CONFIG_URL from GITHUB
 	// ENV vars
 	// Command-line arguments
 
@@ -200,6 +204,18 @@ func InitCliConfig(configAndStacksInfo schema.ConfigAndStacksInfo, processStacks
 		}
 	}
 
+	// Process remote config from a GITHUB URL from the path in ENV var `ATMOS_REMOTE_CONFIG_URL`
+	configFilePath6 := os.Getenv("ATMOS_REMOTE_CONFIG_URL")
+	if len(configFilePath6) > 0 {
+		found, err = processRemoteConfigFile(atmosConfig, configFilePath6, v)
+		if err != nil {
+			return atmosConfig, err
+		}
+		if found {
+			configFound = true
+		}
+	}
+
 	if !configFound {
 		// If `atmos.yaml` not found, use the default config
 		// Set `ATMOS_LOGS_LEVEL` ENV var to "Debug" to see the message about Atmos using the default CLI config
@@ -383,3 +399,34 @@ func processConfigFile(
 
 	return true, nil
 }
+
+// processRemoteConfigFile attempts to download and merge a remote atmos.yaml
+// from a URL. It currently only supports GitHub URLs.
+func processRemoteConfigFile(
+	atmosConfig schema.AtmosConfiguration,
+	rawURL string,
+	v *viper.Viper,
+) (bool, error) {
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil {
+		u.LogWarning(atmosConfig, fmt.Sprintf("Failed to parse remote config URL '%s': %s", rawURL, err.Error()))
+		return false, nil
+	}
+
+	if parsedURL.Scheme != "https" || parsedURL.Host != "github.com" {
+		return false, nil
+	}
+
+	data, err := utils.DownloadFileFromGitHub(rawURL)
+	if err != nil {
+		u.LogWarning(atmosConfig, fmt.Sprintf("Failed to download remote config from GitHub '%s': %s", rawURL, err.Error()))
+		return false, nil
+	}
+
+	err = v.MergeConfig(bytes.NewReader(data))
+	if err != nil {
+		return false, fmt.Errorf("failed to merge remote config from GitHub '%s': %w", rawURL, err)
+	}
+
+	return true, nil
+}

pkg/utils/file_utils.go

@@ -242,3 +242,33 @@ func GetFileNameFromURL(rawURL string) (string, error) {
 	}
 	return fileName, nil
 }
+
+// ParseGitHubURL parses a GitHub URL and returns the owner, repo, file path and branch
+func ParseGitHubURL(rawURL string) (owner, repo, filePath, branch string, err error) {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return "", "", "", "", fmt.Errorf("invalid URL: %w", err)
+	}
+
+	// Expected format: https://github.com/owner/repo/blob/branch/path/to/file
+	parts := strings.Split(u.Path, "/")
+	if len(parts) < 5 || (parts[3] != "blob" && parts[3] != "raw") {
+		return "", "", "", "", fmt.Errorf("invalid GitHub URL format")
+	}
+
+	owner = parts[1]
+	repo = parts[2]
+	branch = parts[4]
+	filePath = strings.Join(parts[5:], "/")
+
+	return owner, repo, filePath, branch, nil
+}
+
+// ParseFilenameFromURL extracts the file name from a URL
+func ParseFilenameFromURL(url string) string {
+	parts := strings.Split(url, "/")
+	if len(parts) == 0 {
+		return ""
+	}
+	return parts[len(parts)-1] // e.g. "dev.yaml"
+}