diff --git a/cmd/terraform.go b/cmd/terraform.go index dfd5ea610..c917fb0ad 100644 --- a/cmd/terraform.go +++ b/cmd/terraform.go @@ -1,6 +1,10 @@ package cmd import ( + "os" + "path/filepath" + "strings" + "github.com/samber/lo" "github.com/spf13/cobra" @@ -42,6 +46,29 @@ var terraformCmd = &cobra.Command{ // Check Atmos configuration checkAtmosConfig() + //Load stack from Github + folderFlag, _ := cmd.Flags().GetString("folder") + if folderFlag != "" && u.IsGithubURL(info.Stack) { + + data, err := u.DownloadFileFromGitHub(info.Stack) + if err != nil { + u.LogErrorAndExit(schema.AtmosConfiguration{}, err) + } + fileName := u.ParseFilenameFromURL(info.Stack) + if fileName == "" { + fileName = "stack.yaml" // fallback + } + localPath := filepath.Join(folderFlag, fileName) + + // Overwrite if it exists + err = os.WriteFile(localPath, data, 0o644) + if err != nil { + u.LogErrorAndExit(schema.AtmosConfiguration{}, err) + } + shortStackName := strings.TrimSuffix(fileName, filepath.Ext(fileName)) + info.Stack = shortStackName + } + err = e.ExecuteTerraform(info) if err != nil { u.LogErrorAndExit(schema.AtmosConfiguration{}, err) @@ -53,5 +80,10 @@ func init() { // https://github.com/spf13/cobra/issues/739 terraformCmd.DisableFlagParsing = true terraformCmd.PersistentFlags().StringP("stack", "s", "", "atmos terraform -s ") + terraformCmd.PersistentFlags().String( + "folder", + "", + "If set, download the remote stack file into this folder, then treat it as a local stack", + ) RootCmd.AddCommand(terraformCmd) } diff --git a/internal/exec/stack_processor_utils.go b/internal/exec/stack_processor_utils.go index d2b9cd9ce..2e341b643 100644 --- a/internal/exec/stack_processor_utils.go +++ b/internal/exec/stack_processor_utils.go @@ -1929,6 +1929,23 @@ func GetFileContent(filePath string) (string, error) { return fmt.Sprintf("%s", existingContent), nil } + //Check if its Github remote URL to single file + // Shoudl be decommised as the direct hook into map does not work + /*parsedURL, err := url.Parse(filePath) // Parse the URL + if err != nil { + u.LogInfo(schema.AtmosConfiguration{}, fmt.Sprintf("Filepath is local: %s", filePath)) + } else { + if parsedURL.Host == "github.com" && parsedURL.Scheme == "https" { + u.LogDebug(schema.AtmosConfiguration{}, fmt.Sprintf("Fetching GitHub source: %s", filePath)) + fileContents, err := u.DownloadFileFromGitHub(filePath) + if err != nil { + return "", fmt.Errorf("failed to download GitHub file: %w", err) + } + getFileContentSyncMap.Store(filePath, fileContents) + return string(fileContents), nil + } + } + */ content, err := os.ReadFile(filePath) if err != nil { return "", err diff --git a/internal/exec/utils.go b/internal/exec/utils.go index cbc435906..bb4f4b268 100644 --- a/internal/exec/utils.go +++ b/internal/exec/utils.go @@ -21,6 +21,7 @@ var ( commonFlags = []string{ "--stack", "-s", + "--folder", cfg.DryRunFlag, cfg.SkipInitFlag, cfg.KubeConfigConfigFlag, diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 95f7dcb0e..5f2ebe518 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -268,6 +268,8 @@ func ExecuteAtmosVendorInternal( return fmt.Errorf("either 'spec.sources' or 'spec.imports' (or both) must be defined in the vendor config file '%s'", vendorConfigFileName) } + var tempDir string + // Process imports and return all sources from all the imports and from `vendor.yaml` sources, _, err := processVendorImports( atmosConfig, @@ -317,6 +319,16 @@ func ExecuteAtmosVendorInternal( ) } + tempDir, err = os.MkdirTemp("", "atmos_vendor_") + if err != nil { + return fmt.Errorf("failed to create temp directory: %w", err) + } + defer func() { + if err := os.RemoveAll(tempDir); err != nil { + u.LogWarning(atmosConfig, fmt.Sprintf("failed to clean up temp directory %s: %v", tempDir, err)) + } + }() + // Allow having duplicate targets in different sources. // This can be used to vendor mixins (from local and remote sources) and write them to the same targets. // TODO: consider adding a flag to `atmos vendor pull` to specify if duplicate targets are allowed or not. @@ -359,7 +371,28 @@ func ExecuteAtmosVendorInternal( return err } - useOciScheme, useLocalFileSystem, sourceIsLocalFile := determineSourceType(&uri, vendorConfigFilePath) + useOciScheme, useLocalFileSystem, sourceIsLocalFile, isGitHubSource := determineSourceType(&uri, vendorConfigFilePath) + + // Handle GitHub source + if isGitHubSource { + if dryRun { + u.LogInfo(atmosConfig, fmt.Sprintf("Dry run: Fetching GitHub source: %s", uri)) + } else { + u.LogDebug(atmosConfig, fmt.Sprintf("Fetching GitHub source: %s", uri)) + fileContents, err := u.DownloadFileFromGitHub(uri) + if err != nil { + return fmt.Errorf("failed to download GitHub file: %w", err) + } + // Save the downloaded file to the existing tempDir + tempGitHubFile := filepath.Join(tempDir, filepath.Base(uri)) + err = os.WriteFile(tempGitHubFile, fileContents, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to save GitHub file to temp location: %w", err) + } + // Update the URI to point to the saved file in the temp directory + uri = tempGitHubFile + } + } // Determine package type var pType pkgType @@ -497,8 +530,8 @@ func shouldSkipSource(s *schema.AtmosVendorSource, component string, tags []stri return (component != "" && s.Component != component) || (len(tags) > 0 && len(lo.Intersect(tags, s.Tags)) == 0) } -func determineSourceType(uri *string, vendorConfigFilePath string) (bool, bool, bool) { - // Determine if the URI is an OCI scheme, a local file, or remote +func determineSourceType(uri *string, vendorConfigFilePath string) (bool, bool, bool, bool) { + // Determine if the URI is an OCI scheme, a local file, a remote GitHub source, or a generic remote useOciScheme := strings.HasPrefix(*uri, "oci://") if useOciScheme { *uri = strings.TrimPrefix(*uri, "oci://") @@ -506,14 +539,26 @@ func determineSourceType(uri *string, vendorConfigFilePath string) (bool, bool, useLocalFileSystem := false sourceIsLocalFile := false + isGitHubSource := false + + // If not OCI, we proceed with checks if !useOciScheme { - if absPath, err := u.JoinAbsolutePathWithPath(vendorConfigFilePath, *uri); err == nil { - uri = &absPath - useLocalFileSystem = true - sourceIsLocalFile = u.FileExists(*uri) + parsedURL, err := url.Parse(*uri) + if err != nil || parsedURL.Scheme == "" || parsedURL.Host == "" { + // Not a valid URL or no host: consider local filesystem + if absPath, err := u.JoinAbsolutePathWithPath(vendorConfigFilePath, *uri); err == nil { + uri = &absPath + useLocalFileSystem = true + sourceIsLocalFile = u.FileExists(*uri) + } + } else { + if parsedURL.Host == "github.com" && parsedURL.Scheme == "https" { + isGitHubSource = true + } } } - return useOciScheme, useLocalFileSystem, sourceIsLocalFile + + return useOciScheme, useLocalFileSystem, sourceIsLocalFile, isGitHubSource } // sanitizeFileName replaces invalid characters and query strings with underscores for Windows. diff --git a/pkg/config/config.go b/pkg/config/config.go index 38420ef56..b73aaebea 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "net/url" "os" "path" "path/filepath" @@ -15,6 +16,7 @@ import ( "github.com/spf13/viper" "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/utils" u "github.com/cloudposse/atmos/pkg/utils" "github.com/cloudposse/atmos/pkg/version" ) @@ -100,6 +102,8 @@ func InitCliConfig(configAndStacksInfo schema.ConfigAndStacksInfo, processStacks // system dir (`/usr/local/etc/atmos` on Linux, `%LOCALAPPDATA%/atmos` on Windows) // home dir (~/.atmos) // current directory + // ENV var ATMOS_CLI_CONFIG_PATH + // ENV var ATMOS_REMOTE_CONFIG_URL from GITHUB // ENV vars // Command-line arguments @@ -200,6 +204,18 @@ func InitCliConfig(configAndStacksInfo schema.ConfigAndStacksInfo, processStacks } } + // Process remote config from a GITHUB URL from the path in ENV var `ATMOS_REMOTE_CONFIG_URL` + configFilePath6 := os.Getenv("ATMOS_REMOTE_CONFIG_URL") + if len(configFilePath6) > 0 { + found, err = processRemoteConfigFile(atmosConfig, configFilePath6, v) + if err != nil { + return atmosConfig, err + } + if found { + configFound = true + } + } + if !configFound { // If `atmos.yaml` not found, use the default config // Set `ATMOS_LOGS_LEVEL` ENV var to "Debug" to see the message about Atmos using the default CLI config @@ -383,3 +399,34 @@ func processConfigFile( return true, nil } + +// processRemoteConfigFile attempts to download and merge a remote atmos.yaml +// from a URL. It currently only supports GitHub URLs. +func processRemoteConfigFile( + atmosConfig schema.AtmosConfiguration, + rawURL string, + v *viper.Viper, +) (bool, error) { + parsedURL, err := url.Parse(rawURL) + if err != nil { + u.LogWarning(atmosConfig, fmt.Sprintf("Failed to parse remote config URL '%s': %s", rawURL, err.Error())) + return false, nil + } + + if parsedURL.Scheme != "https" || parsedURL.Host != "github.com" { + return false, nil + } + + data, err := utils.DownloadFileFromGitHub(rawURL) + if err != nil { + u.LogWarning(atmosConfig, fmt.Sprintf("Failed to download remote config from GitHub '%s': %s", rawURL, err.Error())) + return false, nil + } + + err = v.MergeConfig(bytes.NewReader(data)) + if err != nil { + return false, fmt.Errorf("failed to merge remote config from GitHub '%s': %w", rawURL, err) + } + + return true, nil +} diff --git a/pkg/utils/file_utils.go b/pkg/utils/file_utils.go index ce1509fea..2969c71a5 100644 --- a/pkg/utils/file_utils.go +++ b/pkg/utils/file_utils.go @@ -242,3 +242,33 @@ func GetFileNameFromURL(rawURL string) (string, error) { } return fileName, nil } + +// ParseGitHubURL parses a GitHub URL and returns the owner, repo, file path and branch +func ParseGitHubURL(rawURL string) (owner, repo, filePath, branch string, err error) { + u, err := url.Parse(rawURL) + if err != nil { + return "", "", "", "", fmt.Errorf("invalid URL: %w", err) + } + + // Expected format: https://github.com/owner/repo/blob/branch/path/to/file + parts := strings.Split(u.Path, "/") + if len(parts) < 5 || (parts[3] != "blob" && parts[3] != "raw") { + return "", "", "", "", fmt.Errorf("invalid GitHub URL format") + } + + owner = parts[1] + repo = parts[2] + branch = parts[4] + filePath = strings.Join(parts[5:], "/") + + return owner, repo, filePath, branch, nil +} + +// ParseFilenameFromURL extracts the file name from a URL +func ParseFilenameFromURL(url string) string { + parts := strings.Split(url, "/") + if len(parts) == 0 { + return "" + } + return parts[len(parts)-1] // e.g. "dev.yaml" +} diff --git a/pkg/utils/github_utils.go b/pkg/utils/github_utils.go index 06aef181b..bd08228f4 100644 --- a/pkg/utils/github_utils.go +++ b/pkg/utils/github_utils.go @@ -2,29 +2,94 @@ package utils import ( "context" + "encoding/base64" + "fmt" + "net/url" + "os" "time" "github.com/google/go-github/v59/github" + "golang.org/x/oauth2" ) -// GetLatestGitHubRepoRelease returns the latest release tag for a GitHub repository -func GetLatestGitHubRepoRelease(owner string, repo string) (string, error) { - opt := &github.ListOptions{Page: 1, PerPage: 1} - client := github.NewClient(nil) +// newGitHubClient creates a new GitHub client. If a token is provided, it returns an authenticated client; +// otherwise, it returns an unauthenticated client. +func newGitHubClient(ctx context.Context) *github.Client { + githubToken := os.Getenv("GITHUB_TOKEN") + if githubToken == "" { + return github.NewClient(nil) + } + + // Token found, create an authenticated client + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: githubToken}, + ) + tc := oauth2.NewClient(ctx, ts) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) + return github.NewClient(tc) +} + +// GetLatestGitHubRepoRelease returns the latest release tag for a GitHub repository. +func GetLatestGitHubRepoRelease(owner string, repo string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() + client := newGitHubClient(ctx) + opt := &github.ListOptions{Page: 1, PerPage: 1} + releases, _, err := client.Repositories.ListReleases(ctx, owner, repo, opt) if err != nil { - return "", err + return "", fmt.Errorf("failed to list releases: %w", err) } - if len(releases) > 0 { - latestRelease := releases[0] - latestReleaseTag := *latestRelease.TagName - return latestReleaseTag, nil + if len(releases) > 0 && releases[0].TagName != nil { + return *releases[0].TagName, nil } return "", nil } + +// DownloadFileFromGitHub downloads a file from a GitHub repository using the GitHub API. +func DownloadFileFromGitHub(rawURL string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + owner, repo, filePath, branch, err := ParseGitHubURL(rawURL) + if err != nil { + return nil, fmt.Errorf("failed to parse GitHub URL: %w", err) + } + + client := newGitHubClient(ctx) + + // Get the file content + opt := &github.RepositoryContentGetOptions{Ref: branch} + fileContent, _, _, err := client.Repositories.GetContents(ctx, owner, repo, filePath, opt) + if err != nil { + return nil, fmt.Errorf("failed to get file content from GitHub: %w", err) + } + if fileContent == nil { + return nil, fmt.Errorf("no content returned for the requested file") + } + + // Decode the base64 encoded content + content, err := fileContent.GetContent() + if err != nil { + return nil, fmt.Errorf("failed to get file content: %w", err) + } + data, err := base64.StdEncoding.DecodeString(content) + if err != nil { + // fallback to raw content + data = []byte(content) + } + + return data, nil +} + +// IsGithubURL checks if a URL is a valid GitHub URL. +func IsGithubURL(rawURL string) bool { + parsedURL, err := url.Parse(rawURL) + if err == nil && parsedURL.Host == "github.com" && parsedURL.Scheme == "https" { + return true + } + return false +} diff --git a/pkg/vender/vendor_config_test.go b/pkg/vender/vendor_config_test.go index 3c3005c10..45496434a 100644 --- a/pkg/vender/vendor_config_test.go +++ b/pkg/vender/vendor_config_test.go @@ -143,3 +143,54 @@ spec: assert.Nil(t, err) }) } + +// New separate test function that directly calls ExecuteAtmosVendorInternal +// after setting a GitHub token and reading a vendor.yaml referencing a GitHub source. +func TestExecuteAtmosVendorInternalWithToken(t *testing.T) { + testDir := t.TempDir() + + // Set the GitHub token environment variable + os.Setenv("GITHUB_TOKEN", "my-test-token") + defer os.Unsetenv("GITHUB_TOKEN") + + // Create a vendor.yaml file referencing a GitHub source + vendorYaml := ` +apiVersion: atmos/v1 +kind: AtmosVendorConfig +metadata: + name: test-vendor-config-with-token +spec: + sources: + - component: github-component + source: https://github.com/example/repo/blob/main/test-file.yaml + included_paths: + - "**/*.yaml" + targets: + - vendor/github-component +` + vendorYamlPath := filepath.Join(testDir, "vendor.yaml") + err := os.WriteFile(vendorYamlPath, []byte(vendorYaml), 0644) + assert.Nil(t, err) + + // Initialize CLI configuration for this test + atmosConfig := schema.AtmosConfiguration{ + BasePath: testDir, + Components: schema.Components{ + Terraform: schema.Terraform{ + BasePath: "components/terraform", + }, + }, + } + atmosConfig.Logs.Level = "Trace" + + // Read the vendor config + vendorConfig, exists, foundVendorConfigFile, err := e.ReadAndProcessVendorConfigFile(atmosConfig, vendorYamlPath, true) + assert.Nil(t, err) + assert.True(t, exists) + assert.NotEmpty(t, foundVendorConfigFile) + + // Now call ExecuteAtmosVendorInternal directly + // We use dryRun to avoid downloading and writing files + err = e.ExecuteAtmosVendorInternal(atmosConfig, foundVendorConfigFile, vendorConfig.Spec, "github-component", []string{}, true) + assert.Nil(t, err, "ExecuteAtmosVendorInternal should run without errors using the token in dry-run mode") +}