diff --git a/cloud/storage/storage.go b/cloud/storage/storage.go new file mode 100644 index 0000000..c8e4560 --- /dev/null +++ b/cloud/storage/storage.go @@ -0,0 +1,157 @@ +package storage + +import ( + "strings" + + "gopkg.in/yaml.v3" +) + +// ArtifactRepositoryS3Provider is meant to be used +// by the CLI. CLI will marshal this struct into the correct +// YAML structure for k8s configmap / secret. +type ArtifactRepositoryS3Provider struct { + KeyFormat string `yaml:"keyFormat"` + Bucket string + Endpoint string + Insecure bool + Region string `yaml:"region,omitempty"` + AccessKeySecret ArtifactRepositorySecret `yaml:"accessKeySecret"` + SecretKeySecret ArtifactRepositorySecret `yaml:"secretKeySecret"` + AccessKey string `yaml:"accessKey,omitempty"` + Secretkey string `yaml:"secretKey,omitempty"` +} + +// ArtifactRepositoryGCSProvider is meant to be used +// by the CLI. CLI will marshal this struct into the correct +// YAML structure for k8s configmap / secret. +type ArtifactRepositoryGCSProvider struct { + KeyFormat string `yaml:"keyFormat"` + Bucket string + Endpoint string + Insecure bool + ServiceAccountKey string `yaml:"serviceAccountKey,omitempty"` + ServiceAccountKeySecret ArtifactRepositorySecret `yaml:"serviceAccountKeySecret"` + ServiceAccountJSON string `yaml:"serviceAccountJSON,omitempty"` +} + +// ArtifactRepositoryABSProvider - Azure Blob Storage is meant to be used +// by the CLI. CLI will marshal this struct into the correct +// YAML structure for k8s configmap / secret. +type ArtifactRepositoryABSProvider struct { + KeyFormat string `yaml:"keyFormat"` + Bucket string + Endpoint string + Insecure bool + AccessKeySecret ArtifactRepositorySecret `yaml:"accessKeySecret"` + SecretKeySecret ArtifactRepositorySecret `yaml:"secretKeySecret"` + AccessKey string `yaml:"accessKey,omitempty"` + Secretkey string `yaml:"secretKey,omitempty"` + Container string `yaml:"container"` +} + +// ArtifactRepositoryProvider is used to setup access into AWS Cloud Storage +// or Google Cloud storage. +// - The relevant sub-struct (S3, GCS) is unmarshalled into from the cluster configmap. +// Right now, either the S3 or GCS struct will be filled in. Multiple cloud +// providers are not supported at the same time in params.yaml (manifests deployment). +type ArtifactRepositoryProvider struct { + S3 *ArtifactRepositoryS3Provider `yaml:"s3,omitempty"` + GCS *ArtifactRepositoryGCSProvider `yaml:"gcs,omitempty"` + ABS *ArtifactRepositoryABSProvider `yaml:"abs,omitempty"` +} + +// ArtifactRepositorySecret holds information about a kubernetes Secret. +// - The "key" is the specific key inside the Secret. +// - The "name" is the name of the Secret. +// Usually, this is used to figure out what secret to look into for a specific value. +type ArtifactRepositorySecret struct { + Key string `yaml:"key"` + Name string `yaml:"name"` +} + +// MarshalToYaml is used by the CLI to generate configmaps during deployment +// or build operations. +func (a *ArtifactRepositoryS3Provider) MarshalToYaml() (string, error) { + builder := &strings.Builder{} + encoder := yaml.NewEncoder(builder) + encoder.SetIndent(6) + defer encoder.Close() + err := encoder.Encode(&ArtifactRepositoryProvider{ + S3: &ArtifactRepositoryS3Provider{ + KeyFormat: a.KeyFormat, + Bucket: a.Bucket, + Endpoint: a.Endpoint, + Insecure: a.Insecure, + Region: a.Region, + AccessKeySecret: ArtifactRepositorySecret{ + Name: a.AccessKeySecret.Name, + Key: a.AccessKeySecret.Key, + }, + SecretKeySecret: ArtifactRepositorySecret{ + Name: a.SecretKeySecret.Name, + Key: a.SecretKeySecret.Key, + }, + }, + }) + + if err != nil { + return "", err + } + + return builder.String(), nil +} + +// MarshalToYaml is used by the CLI to generate configmaps during deployment +// or build operations. +func (g *ArtifactRepositoryGCSProvider) MarshalToYaml() (string, error) { + builder := &strings.Builder{} + encoder := yaml.NewEncoder(builder) + encoder.SetIndent(6) + defer encoder.Close() + err := encoder.Encode(&ArtifactRepositoryProvider{ + GCS: &ArtifactRepositoryGCSProvider{ + KeyFormat: g.KeyFormat, + Bucket: g.Bucket, + Endpoint: g.Endpoint, + Insecure: g.Insecure, + ServiceAccountKeySecret: ArtifactRepositorySecret{ + Key: "artifactRepositoryGCSServiceAccountKey", + Name: "onepanel", + }, + }, + }) + + if err != nil { + return "", err + } + + return builder.String(), nil +} + +// FormatKey replaces placeholder values with their actual values and returns this string. +// {{workflow.namespace}} -> namespace +// {{workflow.name}} -> workflowName +// {{pod.name}} -> podName +func (a *ArtifactRepositoryS3Provider) FormatKey(namespace, workflowName, podName string) string { + keyFormat := a.KeyFormat + + keyFormat = strings.Replace(keyFormat, "{{workflow.namespace}}", namespace, -1) + keyFormat = strings.Replace(keyFormat, "{{workflow.name}}", workflowName, -1) + keyFormat = strings.Replace(keyFormat, "{{pod.name}}", podName, -1) + + return keyFormat +} + +// FormatKey replaces placeholder values with their actual values and returns this string. +// {{workflow.namespace}} -> namespace +// {{workflow.name}} -> workflowName +// {{pod.name}} -> podName +func (g *ArtifactRepositoryGCSProvider) FormatKey(namespace, workflowName, podName string) string { + keyFormat := g.KeyFormat + + keyFormat = strings.Replace(keyFormat, "{{workflow.namespace}}", namespace, -1) + keyFormat = strings.Replace(keyFormat, "{{workflow.name}}", workflowName, -1) + keyFormat = strings.Replace(keyFormat, "{{pod.name}}", podName, -1) + + return keyFormat +} diff --git a/cmd/build.go b/cmd/build.go index 0c0edc7..70e1369 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -4,7 +4,7 @@ import ( "encoding/base64" "errors" "fmt" - v1 "github.com/onepanelio/core/pkg" + "github.com/onepanelio/cli/cloud/storage" "golang.org/x/crypto/bcrypt" "io/ioutil" "k8s.io/apimachinery/pkg/util/rand" @@ -83,26 +83,17 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. manifestPath := config.Spec.ManifestsRepo localManifestsCopyPath := filepath.Join(".onepanel", "manifests", "cache") - exists, err := files.Exists(localManifestsCopyPath) - if err != nil { + // Delete the local files if they exist + if err := os.RemoveAll(localManifestsCopyPath); err != nil { return "", err } - if exists { - if err := os.RemoveAll(localManifestsCopyPath); err != nil { - return "", err - } - } - if err := files.CopyDir(manifestPath, localManifestsCopyPath); err != nil { return "", err } localKustomizePath := filepath.Join(localManifestsCopyPath, "kustomization.yaml") - if _, err := files.DeleteIfExists(localKustomizePath); err != nil { - return "", err - } - + // Create will truncate the file if it exists newFile, err := os.Create(localKustomizePath) if err != nil { return "", err @@ -181,7 +172,7 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. } _, artifactRepositoryNode := yamlFile.Get("artifactRepository") - artifactRepositoryConfig := v1.ArtifactRepositoryProvider{} + artifactRepositoryConfig := storage.ArtifactRepositoryProvider{} err = artifactRepositoryNode.Decode(&artifactRepositoryConfig) if err != nil { return "", err @@ -203,11 +194,38 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. } yamlFile.Put("artifactRepositoryProvider", yamlConfigMap) + } else if artifactRepositoryConfig.ABS != nil { + defaultNamespace := yamlFile.GetValue("application.defaultNamespace").Value + artifactRepositoryConfig.S3 = &storage.ArtifactRepositoryS3Provider{ + KeyFormat: artifactRepositoryConfig.ABS.KeyFormat, + Bucket: artifactRepositoryConfig.ABS.Container, + Endpoint: fmt.Sprintf("minio-gateway.%v.svc.cluster.local:9000", defaultNamespace), + Insecure: true, + AccessKeySecret: storage.ArtifactRepositorySecret{ + Key: "artifactRepositoryS3AccessKey", + Name: "$(artifactRepositoryS3AccessKey)", + }, + SecretKeySecret: storage.ArtifactRepositorySecret{ + Key: "artifactRepositoryS3SecretKey", + Name: "$(artifactRepositoryS3SecretKeySecretName)", + }, + } + yamlStr, err := artifactRepositoryConfig.S3.MarshalToYaml() + if err != nil { + return "", err + } + yamlFile.Put("artifactRepositoryProvider", yamlStr) + yamlFile.Put("artifactRepository.s3.accessKey", "placeholder") + yamlFile.Put("artifactRepository.s3.secretKey", "placeholder") + yamlFile.Put("artifactRepository.s3.bucket", "bucket-name") + yamlFile.Put("artifactRepository.s3.endpoint", "minio-gateway.onepanel.svc.cluster.local") + yamlFile.Put("artifactRepository.s3.insecure", "true") } else { return "", errors.New("unsupported artifactRepository configuration") } + flatMap := yamlFile.FlattenToKeyValue(util.LowerCamelCaseFlatMapKeyFormatter) - if err := mapLinkedVars(flatMap, localManifestsCopyPath, &config); err != nil { + if err := mapLinkedVars(flatMap, localManifestsCopyPath, &config, true); err != nil { return "", err } @@ -225,9 +243,8 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. continue } k := keyValArr[0] - /** - Do not include the extra S3 parameters if they are not set in the params.yaml - */ + + // Do not include the extra S3 parameters if they are not set in the params.yaml if artifactRepositoryConfig.S3 == nil { if strings.Contains(k, "S3") { continue @@ -241,15 +258,11 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. if artifactRepositoryConfig.S3 != nil { artifactRepositoryS3AccessKeySecretName, ok := flatMap["artifactRepositoryS3AccessKeySecretName"].(string) if !ok { - if err != nil { - return "", err - } + return "", fmt.Errorf("missing 'artifactRepositoryS3AccessKeySecretName'") } artifactRepositoryS3SecretKeySecretName, ok := flatMap["artifactRepositoryS3SecretKeySecretName"].(string) if !ok { - if err != nil { - return "", err - } + return "", fmt.Errorf("missing 'artifactRepositoryS3SecretKeySecretName'") } artifactRepositoryConfig.S3.AccessKeySecret.Name = artifactRepositoryS3AccessKeySecretName artifactRepositoryConfig.S3.SecretKeySecret.Name = artifactRepositoryS3SecretKeySecretName @@ -263,13 +276,33 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. //Write to env files //workflow-config-map.env //Set extra values for S3 specific configuration. - if artifactRepositoryConfig.S3 != nil && artifactRepositoryConfig.GCS == nil { - if yamlFile.HasKeys("artifactRepository.s3.bucket", "artifactRepository.s3.endpoint", "artifactRepository.s3.insecure", "artifactRepository.s3.region") { - //Clear previous env file + if artifactRepositoryConfig.ABS != nil { + missingKeys := yamlFile.FindMissingKeys("artifactRepository.s3.bucket", "artifactRepository.s3.endpoint", "artifactRepository.s3.insecure") + if len(missingKeys) == 0 { paramsPath := filepath.Join(localManifestsCopyPath, "vars", "workflow-config-map.env") - if _, err := files.DeleteIfExists(paramsPath); err != nil { + //Clear previous env file - create truncates if it exists + paramsFile, err := os.Create(paramsPath) + if err != nil { return "", err } + var stringToWrite = fmt.Sprintf("%v=%v\n%v=%v\n%v=%v\n%v=%v\n", + "artifactRepositoryBucket", flatMap["artifactRepositoryS3Bucket"], + "artifactRepositoryEndpoint", flatMap["artifactRepositoryS3Endpoint"], + "artifactRepositoryInsecure", flatMap["artifactRepositoryS3Insecure"], + ) + _, err = paramsFile.WriteString(stringToWrite) + if err != nil { + return "", err + } + } else { + missingKeysMessage := strings.Join(missingKeys, ", ") + log.Fatalf("missing required values in params.yaml: %v", missingKeysMessage) + } + } else if artifactRepositoryConfig.S3 != nil && artifactRepositoryConfig.GCS == nil { + missingKeys := yamlFile.FindMissingKeys("artifactRepository.s3.bucket", "artifactRepository.s3.endpoint", "artifactRepository.s3.insecure", "artifactRepository.s3.region") + if len(missingKeys) == 0 { + paramsPath := filepath.Join(localManifestsCopyPath, "vars", "workflow-config-map.env") + //Clear previous env file - create truncates if it exists paramsFile, err := os.Create(paramsPath) if err != nil { return "", err @@ -285,16 +318,14 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. return "", err } } else { - log.Fatal("Missing required values in params.yaml, artifactRepository. Check bucket, endpoint, or insecure.") + missingKeysMessage := strings.Join(missingKeys, ", ") + log.Fatalf("missing required values in params.yaml: %v", missingKeysMessage) } } //logging-config-map.env, optional component if yamlFile.HasKeys("logging.image", "logging.volumeStorage") { - //Clear previous env file paramsPath := filepath.Join(localManifestsCopyPath, "vars", "logging-config-map.env") - if _, err := files.DeleteIfExists(paramsPath); err != nil { - return "", err - } + //Clear previous env file paramsFile, err := os.Create(paramsPath) if err != nil { return "", err @@ -312,9 +343,6 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. if yamlFile.HasKey("application.defaultNamespace") { //Clear previous env file paramsPath := filepath.Join(localManifestsCopyPath, "vars", "onepanel-config-map.env") - if _, err := files.DeleteIfExists(paramsPath); err != nil { - return "", err - } paramsFile, err := os.Create(paramsPath) if err != nil { return "", err @@ -322,8 +350,7 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. var stringToWrite = fmt.Sprintf("%v=%v\n", "applicationDefaultNamespace", flatMap["applicationDefaultNamespace"], ) - _, err = paramsFile.WriteString(stringToWrite) - if err != nil { + if _, err := paramsFile.WriteString(stringToWrite); err != nil { return "", err } } else { @@ -333,7 +360,8 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. var secretKeysValues []string artifactRepoSecretPlaceholder := "$(artifactRepositoryProviderSecret)" if yamlFile.HasKey("artifactRepository.s3") { - if yamlFile.HasKeys("artifactRepository.s3.accessKey", "artifactRepository.s3.secretKey") { + missingKeys := yamlFile.FindMissingKeys("artifactRepository.s3.accessKey", "artifactRepository.s3.secretKey") + if len(missingKeys) == 0 { secretKeysValues = append(secretKeysValues, "artifactRepositoryS3AccessKey", "artifactRepositoryS3SecretKey") artifactRepoS3Secret := fmt.Sprintf( @@ -346,7 +374,8 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. return "", err } } else { - log.Fatal("Missing required values in params.yaml, artifactRepository. Check accessKey, or secretKey.") + missingKeysMessage := strings.Join(missingKeys, ", ") + log.Fatalf("Missing required values in params.yaml: %v", missingKeysMessage) } } if yamlFile.HasKey("artifactRepository.gcs") { @@ -356,8 +385,7 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. log.Fatal("artifactRepository.gcs.serviceAccountKey cannot be empty.") } artifactRepoS3Secret := "artifactRepositoryGCSServiceAccountKey: '" + val.Value + "'" - err = replacePlaceholderForSecretManiFile(localManifestsCopyPath, artifactRepoSecretPlaceholder, artifactRepoS3Secret) - if err != nil { + if err := replacePlaceholderForSecretManiFile(localManifestsCopyPath, artifactRepoSecretPlaceholder, artifactRepoS3Secret); err != nil { return "", err } } else { @@ -372,52 +400,8 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template. return "", err } - for _, filePath := range listOfFiles { - manifestFileContent, manifestFileOpenErr := ioutil.ReadFile(filePath) - if manifestFileOpenErr != nil { - return "", manifestFileOpenErr - } - manifestFileContentStr := string(manifestFileContent) - useStr := "" - rawStr := "" - for key := range flatMap { - valueBool, okBool := flatMap[key].(bool) - if okBool { - useStr = strconv.FormatBool(valueBool) - rawStr = strconv.FormatBool(valueBool) - } else { - valueInt, okInt := flatMap[key].(int) - if okInt { - useStr = "\"" + strconv.FormatInt(int64(valueInt), 10) + "\"" - rawStr = strconv.FormatInt(int64(valueInt), 10) - } else { - valueStr, ok := flatMap[key].(string) - if !ok { - log.Fatal("Unrecognized value in flatmap. Check type assertions.") - } - useStr = valueStr - rawStr = valueStr - } - } - oldString := "$(" + key + ")" - if strings.Contains(manifestFileContentStr, key) { - manifestFileContentStr = strings.Replace(manifestFileContentStr, oldString, useStr, -1) - } - oldRawString := "$raw(" + key + ")" - if strings.Contains(manifestFileContentStr, key) { - manifestFileContentStr = strings.Replace(manifestFileContentStr, oldRawString, rawStr, -1) - } - - oldBase64String := "$base64(" + key + ")" - if strings.Contains(manifestFileContentStr, key) { - base64Value := base64.StdEncoding.EncodeToString([]byte(rawStr)) - manifestFileContentStr = strings.Replace(manifestFileContentStr, oldBase64String, base64Value, -1) - } - } - writeFileErr := ioutil.WriteFile(filePath, []byte(manifestFileContentStr), 0644) - if writeFileErr != nil { - return "", writeFileErr - } + if err := replaceVariables(flatMap, listOfFiles); err != nil { + return "", err } //Update the values in those files @@ -571,38 +555,47 @@ func generateMetalLbAddresses(nodePoolData []*yaml2.Node) string { // mapLinkedVars goes through the `default-vars.yaml` files which map variables from already existing variables // and set those variable values. If the value is already in the mapping, it is not mapped to the default. -func mapLinkedVars(mapping map[string]interface{}, manifestPath string, config *opConfig.Config) error { - linkVars := false +func mapLinkedVars(mapping map[string]interface{}, manifestPath string, config *opConfig.Config, replace bool) error { + paths := make([]string, 0) + pathsAdded := make(map[string]bool) + for _, component := range config.Spec.Components { - if strings.Contains(component, "modeldb") { - linkVars = true - break + if !pathsAdded["modeldb"] && strings.Contains(component, "modeldb") { + paths = append(paths, filepath.Join(manifestPath, "modeldb", "base", "default-vars.yaml")) + pathsAdded["modeldb"] = true } } - if !linkVars { - return nil - } - - modelDBMapping, err := util.LoadDynamicYamlFromFile(filepath.Join(manifestPath, "modeldb", "base", "default-vars.yaml")) - if err != nil { - return err + for _, component := range config.Spec.Overlays { + if !pathsAdded["artifact-repository"] && strings.Contains(component, filepath.Join("artifact-repository", "overlays", "abs")) { + paths = append(paths, filepath.Join(manifestPath, "common", "artifact-repository", "overlays", "abs", "default-vars.yaml")) + pathsAdded["artifact-repository"] = true + } } - flatMappedVars := modelDBMapping.Flatten(util.LowerCamelCaseFlatMapKeyFormatter) - for key, valueNode := range flatMappedVars { - // Skip if key already exists - if _, ok := mapping[key]; ok { - continue + for _, path := range paths { + loadedMapping, err := util.LoadDynamicYamlFromFile(path) + if err != nil { + return err } - valueKey := util.LowerCamelCaseStringFormat(valueNode.Value.Value, ".") - value, ok := mapping[valueKey] - if !ok { - return fmt.Errorf("unknown key %v", valueKey) - } + flatMappedVars := loadedMapping.Flatten(util.LowerCamelCaseFlatMapKeyFormatter) + for key, valueNode := range flatMappedVars { + // Skip if key already exists + if !replace { + if _, ok := mapping[key]; ok { + continue + } + } + + valueKey := util.LowerCamelCaseStringFormat(valueNode.Value.Value, ".") + value, ok := mapping[valueKey] + if !ok { + return fmt.Errorf("unknown key %v", valueKey) + } - mapping[key] = value + mapping[key] = value + } } return nil @@ -625,3 +618,66 @@ func HumanizeKustomizeError(err error) string { return fmt.Sprintf("Error generating result: %v", err.Error()) } + +// replaceVariable will go through the variables in flatMap and replace any instances of it in fileContent +// the resulting modified content is returned +func replaceVariable(flatMap map[string]interface{}, fileContent []byte) []byte { + manifestFileContentStr := string(fileContent) + useStr := "" + rawStr := "" + + for key := range flatMap { + valueBool, okBool := flatMap[key].(bool) + if okBool { + useStr = strconv.FormatBool(valueBool) + rawStr = strconv.FormatBool(valueBool) + } else { + valueInt, okInt := flatMap[key].(int) + if okInt { + useStr = "\"" + strconv.FormatInt(int64(valueInt), 10) + "\"" + rawStr = strconv.FormatInt(int64(valueInt), 10) + } else { + valueStr, ok := flatMap[key].(string) + if !ok { + log.Fatal("Unrecognized value in flatmap. Check type assertions.") + } + useStr = valueStr + rawStr = valueStr + } + } + oldString := "$(" + key + ")" + if strings.Contains(manifestFileContentStr, key) { + manifestFileContentStr = strings.Replace(manifestFileContentStr, oldString, useStr, -1) + } + oldRawString := "$raw(" + key + ")" + if strings.Contains(manifestFileContentStr, key) { + manifestFileContentStr = strings.Replace(manifestFileContentStr, oldRawString, rawStr, -1) + } + + oldBase64String := "$base64(" + key + ")" + if strings.Contains(manifestFileContentStr, key) { + base64Value := base64.StdEncoding.EncodeToString([]byte(rawStr)) + manifestFileContentStr = strings.Replace(manifestFileContentStr, oldBase64String, base64Value, -1) + } + } + + return []byte(manifestFileContentStr) +} + +// replaceVariables will go through the variables in flatMap and replace any instances of it in any of the files in filePaths +// the file content is replaced, no backups are made. +func replaceVariables(flatMap map[string]interface{}, filePaths []string) error { + for _, filePath := range filePaths { + manifestFileContent, manifestFileOpenErr := ioutil.ReadFile(filePath) + if manifestFileOpenErr != nil { + return manifestFileOpenErr + } + + manifestFileContentStr := replaceVariable(flatMap, manifestFileContent) + if err := ioutil.WriteFile(filePath, manifestFileContentStr, 0644); err != nil { + return err + } + } + + return nil +} diff --git a/cmd/init.go b/cmd/init.go index 39a2e23..19e8e24 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -21,6 +21,7 @@ const ( manifestsFilePath = ".onepanel/manifests" artifactRepositoryProviderS3 = "s3" artifactRepositoryProviderGcs = "gcs" + artifactRepositoryProviderAbs = "abs" ) var ( @@ -149,6 +150,8 @@ var initCmd = &cobra.Command{ return } + bld.AddOverlayContender(ArtifactRepositoryProvider) + if err := addCloudProviderToManifestBuilder(Provider, bld); err != nil { log.Printf("[error] Adding Cloud Provider: %v", err.Error()) return @@ -293,7 +296,7 @@ func init() { initCmd.Flags().StringVarP(&Provider, "provider", "p", "", "Cloud provider. Valid values: aks, gke, eks") initCmd.Flags().StringVarP(&DNS, "dns-provider", "d", "", "Provider for DNS. Valid values: azuredns, clouddns (google), cloudflare, route53") - initCmd.Flags().StringVarP(&ArtifactRepositoryProvider, "artifact-repository-provider", "", "", "Object storage provider for storing artifacts. Valid value: s3") + initCmd.Flags().StringVarP(&ArtifactRepositoryProvider, "artifact-repository-provider", "", "", "Object storage provider for storing artifacts. Valid value: s3, abs") initCmd.Flags().StringVarP(&ConfigurationFilePath, "config", "c", "config.yaml", "File path of the resulting config file") initCmd.Flags().StringVarP(&ParametersFilePath, "params", "e", "params.yaml", "File path of the resulting parameters file") initCmd.Flags().BoolVarP(&EnableEFKLogging, "enable-efk-logging", "", false, "Enable Elasticsearch, Fluentd and Kibana (EFK) logging") @@ -363,14 +366,16 @@ func validateProvider(prov string) error { func validateArtifactRepositoryProvider(arRepoProv string) error { if arRepoProv == "" { - return errors.New("artifact-repository-provider flag is required. Valid value: s3") + return errors.New("artifact-repository-provider flag is required. Valid value: s3, abs") } if arRepoProv == artifactRepositoryProviderS3 || - arRepoProv == artifactRepositoryProviderGcs { + arRepoProv == artifactRepositoryProviderGcs || + arRepoProv == artifactRepositoryProviderAbs { return nil } - return fmt.Errorf("'%v' is not a valid --artifact-repository-provider value. Valid value: s3", arRepoProv) + + return fmt.Errorf("'%v' is not a valid --artifact-repository-provider value. Valid value: s3, abs", arRepoProv) } func validateDNS(dns string) error { @@ -442,7 +447,7 @@ func addDNSProviderToManifestBuilder(dns string, builder *manifest.Builder) erro } func removeUneededArtifactRepositoryProviders(mergedParams *util.DynamicYaml) { - artifactRepoProviders := []string{artifactRepositoryProviderS3, artifactRepositoryProviderGcs} + artifactRepoProviders := []string{artifactRepositoryProviderS3, artifactRepositoryProviderGcs, artifactRepositoryProviderAbs} var nodeKeyStr string for _, artRepoProv := range artifactRepoProviders { if ArtifactRepositoryProvider == artRepoProv { diff --git a/go.mod b/go.mod index af589c8..72c9795 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/google/gofuzz v1.1.0 // indirect github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 github.com/mitchellh/go-homedir v1.1.0 - github.com/onepanelio/core v0.12.0 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v0.0.5 github.com/spf13/viper v1.5.0 @@ -21,5 +20,4 @@ require ( k8s.io/kubectl v0.17.3 k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab // indirect sigs.k8s.io/kustomize/api v0.3.2 - ) diff --git a/util/dynamic_yaml.go b/util/dynamic_yaml.go index 342b3bf..39cfc89 100644 --- a/util/dynamic_yaml.go +++ b/util/dynamic_yaml.go @@ -202,6 +202,19 @@ func (d *DynamicYaml) HasKeys(keys ...string) bool { return true } +// FindMissingKeys will return an array of the keys that are not in the manifest +func (d *DynamicYaml) FindMissingKeys(keys ...string) []string { + missing := make([]string, 0) + + for _, key := range keys { + if !d.HasKey(key) { + missing = append(missing, key) + } + } + + return missing +} + func createMappingYamlNode() *yaml.Node { return &yaml.Node{ Kind: yaml.MappingNode,