diff --git a/README.md b/README.md index cc1b700..738517c 100644 --- a/README.md +++ b/README.md @@ -152,8 +152,79 @@ The order of configuration precedence is as follows: There are some configuration examples in the `examples` directory of this repo. +2021/01/22 11:34:10 (DEBUG) Using config file: /lagoon/.lagoon-sync +2021/01/22 11:34:10 (DEBUG) Config that will be used for sync: + { + "Config": { + "DbHostname": "$MARIADB_HOST", + "DbUsername": "$MARIADB_USERNAME", + "DbPassword": "$MARIADB_PASSWORD", + "DbPort": "$MARIADB_PORT", + "DbDatabase": "$MARIADB_DATABASE", + ... + +To recap, the configuration files that can be used by default, in order of priority when available are: +* /lagoon/.lagoon-sync-defaults +* /lagoon/.lagoon-sync +* .lagoon.yml + +### Custom synchers + +It's possible to extend lagoon-sync to define your own sync processes. As lagoon-sync is essentially a +script runner that runs commands on target and source systems, as well as transferring data between the two systems, +it's possible to define commands that generate the transfer resource and consume it on the target. + +For instance, if you have [mtk](https://github.com/skpr/mtk) set up on the target machine, it should be possible to +define a custom syncher that makes use of mtk to generate a sanitized DB dump on the source, and then use mysql to +import it on the target. + +This is done by defining three things: +* The transfer resource name (what file is going to be synced across the network) - in this case let's call it "/tmp/dump.sql" +* The command(s) to run on the source +* The command(s) to run target + +``` +lagoon-sync: + mtkdump: + transfer-resource: "/tmp/dump.sql" + source: + commands: + - "mtk-dump > {{ .transferResource }}" + target: + commands: + - "mysql -h${MARIADB_HOST:-mariadb} -u${MARIADB_USERNAME:-drupal} -p${MARIADB_PASSWORD:-drupal} -P${MARIADB_PORT:-3306} ${MARIADB_DATABASE:-drupal} < {{ .transfer-resource }}" +``` + +This can then be called by running the following: +``` +lagoon-sync sync mtkdump -p -e +``` + +### Custom configuration files +If you don't want your configuration file inside `/lagoon` and want to give it another name then you can define a custom file and tell sync to use that by providing the file path. This can be done with `--config` flag such as:Config files that can be used in order of priority: +- .lagoon-sync-defaults _(no yaml ext neeeded)_ +- .lagoon-sync _(no yaml ext neeeded)_ +- .lagoon.yml _Main config file - path can be given as an argument with `--config`, default is `.lagoon.yml`_ +å +``` +$ lagoon-sync sync mariadb -p mysite-com -e dev --config=/app/.lagoon-sync --show-debug + +2021/01/22 11:43:50 (DEBUG) Using config file: /app/.lagoon-sync +``` + +You can also use an environment variable to set the config sync path with either `LAGOON_SYNC_PATH` or `LAGOON_SYNC_DEFAULTS_PATH`. + +``` +$ LAGOON_SYNC_PATH=/app/.lagoon-sync lagoon-sync sync mariadb -p mysite-com -e dev --show-debug + +2021/01/22 11:46:42 (DEBUG) LAGOON_SYNC_PATH env var found: /app/.lagoon-sync +2021/01/22 11:46:42 (DEBUG) Using config file: /app/.lagoon-sync +``` + + To double check which config file is loaded you can also run the `lagoon-sync config` command. + ### Example sync config overrides ``` lagoon-sync: diff --git a/cmd/sync.go b/cmd/sync.go index e16ee15..1d37b69 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -109,7 +109,11 @@ func syncCommandRun(cmd *cobra.Command, args []string) { // GetSyncersForTypeFromConfigRoot will return a prepared mariadb syncher object) lagoonSyncer, err := synchers.GetSyncerForTypeFromConfigRoot(SyncerType, configRoot) if err != nil { - utils.LogFatalError(err.Error(), nil) + // Let's ask the custom syncer if this will work, if so, we fall back on it ... + lagoonSyncer, err = synchers.GetCustomSync(configRoot, SyncerType) + if err != nil { + utils.LogFatalError(err.Error(), nil) + } } if ProjectName == "" { diff --git a/synchers/custom.go b/synchers/custom.go new file mode 100644 index 0000000..a8a151e --- /dev/null +++ b/synchers/custom.go @@ -0,0 +1,174 @@ +package synchers + +import ( + "errors" + "fmt" + "reflect" + + "github.com/uselagoon/lagoon-sync/utils" +) + +type BaseCustomSyncCommands struct { + Commands []string `yaml:"commands"` +} + +type BaseCustomSync struct { +} + +func (customConfig *BaseCustomSync) setDefaults() { + // Defaults don't make sense here, so noop +} + +type CustomSyncRoot struct { + TransferResource string `yaml:"transfer-resource"` + Source BaseCustomSyncCommands `yaml:"source"` + Target BaseCustomSyncCommands `yaml:"target"` +} + +func (m CustomSyncRoot) SetTransferResource(transferResourceName string) error { + m.TransferResource = transferResourceName + return nil +} + +// Init related types and functions follow + +type CustomSyncPlugin struct { + isConfigEmpty bool + CustomRoot string +} + +func (m BaseCustomSync) IsBaseCustomStructureEmpty() bool { + return reflect.DeepEqual(m, BaseCustomSync{}) +} + +func (m CustomSyncPlugin) GetPluginId() string { + if m.CustomRoot != "" { + return m.CustomRoot + } + return "custom" +} + +func GetCustomSync(configRoot SyncherConfigRoot, syncerName string) (Syncer, error) { + + m := CustomSyncPlugin{ + CustomRoot: syncerName, + } + + ret, err := m.UnmarshallYaml(configRoot) + if err != nil { + return CustomSyncRoot{}, err + } + + return ret, nil +} + +func (m CustomSyncPlugin) UnmarshallYaml(root SyncherConfigRoot) (Syncer, error) { + custom := CustomSyncRoot{} + + // Use 'environment-defaults' if present + envVars := root.Prerequisites + var configMap interface{} + + configMap = root.LagoonSync[m.GetPluginId()] + + if envVars == nil { + // Use 'lagoon-sync' yaml as override if env vars are not available + configMap = root.LagoonSync[m.GetPluginId()] + } + + // If config from active config file is empty, then use defaults + if configMap == nil { + utils.LogDebugInfo("Active syncer config is empty, so using defaults", custom) + } + + // unmarshal environment variables as defaults + err := UnmarshalIntoStruct(configMap, &custom) + if err != nil { + + } + + if len(root.LagoonSync) != 0 { + _ = UnmarshalIntoStruct(configMap, &custom) + utils.LogDebugInfo("Config that will be used for sync", custom) + } + + lagoonSyncer, _ := custom.PrepareSyncer() + + if custom.TransferResource == "" { + return lagoonSyncer, errors.New("Transfer resource MUST be set on custom syncher definition") + } + + return lagoonSyncer, nil +} + +func init() { + RegisterSyncer(CustomSyncPlugin{}) +} + +func (m CustomSyncRoot) IsInitialized() (bool, error) { + return true, nil +} + +// Sync related functions follow +func (root CustomSyncRoot) PrepareSyncer() (Syncer, error) { + //root.TransferId = strconv.FormatInt(time.Now().UnixNano(), 10) + return root, nil +} + +func (root CustomSyncRoot) GetPrerequisiteCommand(environment Environment, command string) SyncCommand { + lagoonSyncBin, _ := utils.FindLagoonSyncOnEnv() + + return SyncCommand{ + command: fmt.Sprintf("{{ .bin }} {{ .command }} || true"), + substitutions: map[string]interface{}{ + "bin": lagoonSyncBin, + "command": command, + }, + } +} + +func (root CustomSyncRoot) GetRemoteCommand(sourceEnvironment Environment) []SyncCommand { + + transferResource := root.GetTransferResource(sourceEnvironment) + + ret := []SyncCommand{} + + substitutions := map[string]interface{}{ + "transferResource": transferResource.Name, + } + + for _, c := range root.Source.Commands { + ret = append(ret, generateSyncCommand(c, substitutions)) + } + + return ret +} + +func (m CustomSyncRoot) GetLocalCommand(targetEnvironment Environment) []SyncCommand { + transferResource := m.GetTransferResource(targetEnvironment) + + ret := []SyncCommand{} + + substitutions := map[string]interface{}{ + "transferResource": transferResource.Name, + } + + for _, c := range m.Target.Commands { + ret = append(ret, generateSyncCommand(c, substitutions)) + } + + return ret +} + +func (m CustomSyncRoot) GetFilesToCleanup(environment Environment) []string { + transferResource := m.GetTransferResource(environment) + return []string{ + transferResource.Name, + } +} + +func (m CustomSyncRoot) GetTransferResource(environment Environment) SyncerTransferResource { + return SyncerTransferResource{ + Name: m.TransferResource, + IsDirectory: false} +} diff --git a/synchers/custom_test.go b/synchers/custom_test.go new file mode 100644 index 0000000..6f824d2 --- /dev/null +++ b/synchers/custom_test.go @@ -0,0 +1,153 @@ +package synchers + +import ( + "reflect" + "testing" +) + +func TestCustomSyncPlugin_UnmarshallYaml(t *testing.T) { + type fields struct { + isConfigEmpty bool + } + type args struct { + root SyncherConfigRoot + } + tests := []struct { + name string + fields fields + args args + want Syncer + wantErr bool + }{ + { + name: "simple unmarshalling", + fields: fields{isConfigEmpty: false}, + args: args{ + root: SyncherConfigRoot{ + Project: "", + LagoonSync: map[string]interface{}{ + "custom": CustomSyncRoot{ + TransferResource: "testing", + Source: BaseCustomSyncCommands{Commands: []string{"first"}}, + Target: BaseCustomSyncCommands{Commands: []string{"second"}}, + }, + }, + Prerequisites: nil, + }, + }, + want: CustomSyncRoot{ + TransferResource: "testing", + Source: BaseCustomSyncCommands{Commands: []string{"first"}}, + Target: BaseCustomSyncCommands{Commands: []string{"second"}}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := CustomSyncPlugin{ + isConfigEmpty: tt.fields.isConfigEmpty, + } + got, err := m.UnmarshallYaml(tt.args.root) + if (err != nil) != tt.wantErr { + t.Errorf("UnmarshallYaml() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("UnmarshallYaml() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetCustomSync(t *testing.T) { + type args struct { + configRoot SyncherConfigRoot + syncerName string + } + tests := []struct { + name string + args args + want Syncer + wantErr bool + }{ + { + name: "simple unmarshalling with custom root name", + //fields: fields{isConfigEmpty: false}, + args: args{ + syncerName: "customroot", + configRoot: SyncherConfigRoot{ + Project: "", + LagoonSync: map[string]interface{}{ + "customroot": CustomSyncRoot{ + TransferResource: "testing", + Source: BaseCustomSyncCommands{Commands: []string{"first"}}, + Target: BaseCustomSyncCommands{Commands: []string{"second"}}, + }, + }, + Prerequisites: nil, + }, + }, + want: CustomSyncRoot{ + TransferResource: "testing", + Source: BaseCustomSyncCommands{Commands: []string{"first"}}, + Target: BaseCustomSyncCommands{Commands: []string{"second"}}, + }, + }, + { + name: "simple unmarshalling with multiple commands", + //fields: fields{isConfigEmpty: false}, + args: args{ + syncerName: "customroot", + configRoot: SyncherConfigRoot{ + Project: "", + LagoonSync: map[string]interface{}{ + "customroot": CustomSyncRoot{ + TransferResource: "testing", + Source: BaseCustomSyncCommands{Commands: []string{"first of one", "second of one"}}, + Target: BaseCustomSyncCommands{Commands: []string{"first of two", "second of two"}}, + }, + }, + Prerequisites: nil, + }, + }, + want: CustomSyncRoot{ + TransferResource: "testing", + Source: BaseCustomSyncCommands{Commands: []string{"first of one", "second of one"}}, + Target: BaseCustomSyncCommands{Commands: []string{"first of two", "second of two"}}, + }, + }, + { + name: "Fails because of empty transfer resource", + //fields: fields{isConfigEmpty: false}, + wantErr: true, + args: args{ + syncerName: "customroot", + configRoot: SyncherConfigRoot{ + Project: "", + LagoonSync: map[string]interface{}{ + "customroot": CustomSyncRoot{ + TransferResource: "", + Source: BaseCustomSyncCommands{Commands: []string{"first of one", "second of one"}}, + Target: BaseCustomSyncCommands{Commands: []string{"first of two", "second of two"}}, + }, + }, + Prerequisites: nil, + }, + }, + want: CustomSyncRoot{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + got, errs := GetCustomSync(tt.args.configRoot, tt.args.syncerName) + if (errs != nil) != tt.wantErr { + t.Errorf("GetCustomSync() error = %v, wantErr %v", errs, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetCustomSync() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/test-resources/custom-syncher/test1/.lagoon.yml b/test-resources/custom-syncher/test1/.lagoon.yml new file mode 100644 index 0000000..086d8c4 --- /dev/null +++ b/test-resources/custom-syncher/test1/.lagoon.yml @@ -0,0 +1,75 @@ +# Example .lagoon.yml file with lagoon-sync config added which is used by the sync tool. +docker-compose-yaml: docker-compose.yml + +project: "lagoon-sync" + +lagoon-sync: + custom: + transfer-resource: "/tmp/hi.txt" + source: + commands: + - "echo 'this should be transferred' > {{ .transferResource }}" + target: + commands: + - "cat /tmp/{{ .transferResource }}" + mariadb: + config: + hostname: "$MARIADB_HOST" + username: "$MARIADB_USERNAME" + password: "$MARIADB_PASSWORD" + port: "$MARIADB_PORT" + database: "$MARIADB_DATABASE" + ignore-table: + - "table_to_ignore" + ignore-table-data: + - "cache_data" + - "cache_menu" + local: + config: + hostname: "mariadb" + username: "drupal" + password: "drupal" + port: "3306" + database: "drupal" + postgres: + config: + hostname: "$POSTGRES_HOST" + username: "$POSTGRES_USERNAME" + password: "$POSTGRES_PASSWORD" + port: "5432" + database: "$POSTGRES_DATABASE" + exclude-table: + - "table_to_ignore" + exclude-table-data: + - "cache_data" + - "cache_menu" + local: + config: + hostname: "postgres" + username: "drupal" + password: "drupal" + port: "3306" + database: "drupal" + mongodb: + config: + hostname: "$MONGODB_HOST" + port: "$MONGODB_SERVICE_PORT" + database: "MONGODB_DATABASE" + local: + config: + hostname: "$MONGODB_HOST" + port: "27017" + database: "local" + files: + config: + sync-directory: "/app/web/sites/default/files" + local: + config: + sync-directory: "/app/web/sites/default/files" + drupalconfig: + config: + syncpath: "./config/sync" + local: + overrides: + config: + syncpath: "./config/sync"