Skip to content

Commit

Permalink
Adds custom command type (#73)
Browse files Browse the repository at this point in the history
* Adds custom command type and docs

* Support multiple commands for custom synchers


---------

Co-authored-by: Tim Clifford <tim.clifford@amazee.com>
  • Loading branch information
bomoko and Tim Clifford authored Oct 30, 2023
1 parent 604ee55 commit 6a83f78
Show file tree
Hide file tree
Showing 5 changed files with 478 additions and 1 deletion.
71 changes: 71 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -152,8 +152,79 @@ The order of configuration precedence is as follows:

There are some configuration examples in the `examples` directory of this repo.

2021/01/22 11:34:10 (DEBUG) Using config file: /lagoon/.lagoon-sync
2021/01/22 11:34:10 (DEBUG) Config that will be used for sync:
{
"Config": {
"DbHostname": "$MARIADB_HOST",
"DbUsername": "$MARIADB_USERNAME",
"DbPassword": "$MARIADB_PASSWORD",
"DbPort": "$MARIADB_PORT",
"DbDatabase": "$MARIADB_DATABASE",
...

To recap, the configuration files that can be used by default, in order of priority when available are:
* /lagoon/.lagoon-sync-defaults
* /lagoon/.lagoon-sync
* .lagoon.yml

### Custom synchers

It's possible to extend lagoon-sync to define your own sync processes. As lagoon-sync is essentially a
script runner that runs commands on target and source systems, as well as transferring data between the two systems,
it's possible to define commands that generate the transfer resource and consume it on the target.

For instance, if you have [mtk](https://github.com/skpr/mtk) set up on the target machine, it should be possible to
define a custom syncher that makes use of mtk to generate a sanitized DB dump on the source, and then use mysql to
import it on the target.

This is done by defining three things:
* The transfer resource name (what file is going to be synced across the network) - in this case let's call it "/tmp/dump.sql"
* The command(s) to run on the source
* The command(s) to run target

```
lagoon-sync:
mtkdump:
transfer-resource: "/tmp/dump.sql"
source:
commands:
- "mtk-dump > {{ .transferResource }}"
target:
commands:
- "mysql -h${MARIADB_HOST:-mariadb} -u${MARIADB_USERNAME:-drupal} -p${MARIADB_PASSWORD:-drupal} -P${MARIADB_PORT:-3306} ${MARIADB_DATABASE:-drupal} < {{ .transfer-resource }}"
```

This can then be called by running the following:
```
lagoon-sync sync mtkdump -p <SOURCE_PROJECT> -e <SOURCE_ENVIRONMENT>
```

### Custom configuration files
If you don't want your configuration file inside `/lagoon` and want to give it another name then you can define a custom file and tell sync to use that by providing the file path. This can be done with `--config` flag such as:Config files that can be used in order of priority:
- .lagoon-sync-defaults _(no yaml ext neeeded)_
- .lagoon-sync _(no yaml ext neeeded)_
- .lagoon.yml _Main config file - path can be given as an argument with `--config`, default is `.lagoon.yml`_
å
```
$ lagoon-sync sync mariadb -p mysite-com -e dev --config=/app/.lagoon-sync --show-debug
2021/01/22 11:43:50 (DEBUG) Using config file: /app/.lagoon-sync
```

You can also use an environment variable to set the config sync path with either `LAGOON_SYNC_PATH` or `LAGOON_SYNC_DEFAULTS_PATH`.

```
$ LAGOON_SYNC_PATH=/app/.lagoon-sync lagoon-sync sync mariadb -p mysite-com -e dev --show-debug
2021/01/22 11:46:42 (DEBUG) LAGOON_SYNC_PATH env var found: /app/.lagoon-sync
2021/01/22 11:46:42 (DEBUG) Using config file: /app/.lagoon-sync
```


To double check which config file is loaded you can also run the `lagoon-sync config` command.


### Example sync config overrides
```
lagoon-sync:
Expand Down
6 changes: 5 additions & 1 deletion cmd/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,11 @@ func syncCommandRun(cmd *cobra.Command, args []string) {
// GetSyncersForTypeFromConfigRoot will return a prepared mariadb syncher object)
lagoonSyncer, err := synchers.GetSyncerForTypeFromConfigRoot(SyncerType, configRoot)
if err != nil {
utils.LogFatalError(err.Error(), nil)
// Let's ask the custom syncer if this will work, if so, we fall back on it ...
lagoonSyncer, err = synchers.GetCustomSync(configRoot, SyncerType)
if err != nil {
utils.LogFatalError(err.Error(), nil)
}
}

if ProjectName == "" {
Expand Down
174 changes: 174 additions & 0 deletions synchers/custom.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
package synchers

import (
"errors"
"fmt"
"reflect"

"github.com/uselagoon/lagoon-sync/utils"
)

type BaseCustomSyncCommands struct {
Commands []string `yaml:"commands"`
}

type BaseCustomSync struct {
}

func (customConfig *BaseCustomSync) setDefaults() {
// Defaults don't make sense here, so noop
}

type CustomSyncRoot struct {
TransferResource string `yaml:"transfer-resource"`
Source BaseCustomSyncCommands `yaml:"source"`
Target BaseCustomSyncCommands `yaml:"target"`
}

func (m CustomSyncRoot) SetTransferResource(transferResourceName string) error {
m.TransferResource = transferResourceName
return nil
}

// Init related types and functions follow

type CustomSyncPlugin struct {
isConfigEmpty bool
CustomRoot string
}

func (m BaseCustomSync) IsBaseCustomStructureEmpty() bool {
return reflect.DeepEqual(m, BaseCustomSync{})
}

func (m CustomSyncPlugin) GetPluginId() string {
if m.CustomRoot != "" {
return m.CustomRoot
}
return "custom"
}

func GetCustomSync(configRoot SyncherConfigRoot, syncerName string) (Syncer, error) {

m := CustomSyncPlugin{
CustomRoot: syncerName,
}

ret, err := m.UnmarshallYaml(configRoot)
if err != nil {
return CustomSyncRoot{}, err
}

return ret, nil
}

func (m CustomSyncPlugin) UnmarshallYaml(root SyncherConfigRoot) (Syncer, error) {
custom := CustomSyncRoot{}

// Use 'environment-defaults' if present
envVars := root.Prerequisites
var configMap interface{}

configMap = root.LagoonSync[m.GetPluginId()]

if envVars == nil {
// Use 'lagoon-sync' yaml as override if env vars are not available
configMap = root.LagoonSync[m.GetPluginId()]
}

// If config from active config file is empty, then use defaults
if configMap == nil {
utils.LogDebugInfo("Active syncer config is empty, so using defaults", custom)
}

// unmarshal environment variables as defaults
err := UnmarshalIntoStruct(configMap, &custom)
if err != nil {

}

if len(root.LagoonSync) != 0 {
_ = UnmarshalIntoStruct(configMap, &custom)
utils.LogDebugInfo("Config that will be used for sync", custom)
}

lagoonSyncer, _ := custom.PrepareSyncer()

if custom.TransferResource == "" {
return lagoonSyncer, errors.New("Transfer resource MUST be set on custom syncher definition")
}

return lagoonSyncer, nil
}

func init() {
RegisterSyncer(CustomSyncPlugin{})
}

func (m CustomSyncRoot) IsInitialized() (bool, error) {
return true, nil
}

// Sync related functions follow
func (root CustomSyncRoot) PrepareSyncer() (Syncer, error) {
//root.TransferId = strconv.FormatInt(time.Now().UnixNano(), 10)
return root, nil
}

func (root CustomSyncRoot) GetPrerequisiteCommand(environment Environment, command string) SyncCommand {
lagoonSyncBin, _ := utils.FindLagoonSyncOnEnv()

return SyncCommand{
command: fmt.Sprintf("{{ .bin }} {{ .command }} || true"),
substitutions: map[string]interface{}{
"bin": lagoonSyncBin,
"command": command,
},
}
}

func (root CustomSyncRoot) GetRemoteCommand(sourceEnvironment Environment) []SyncCommand {

transferResource := root.GetTransferResource(sourceEnvironment)

ret := []SyncCommand{}

substitutions := map[string]interface{}{
"transferResource": transferResource.Name,
}

for _, c := range root.Source.Commands {
ret = append(ret, generateSyncCommand(c, substitutions))
}

return ret
}

func (m CustomSyncRoot) GetLocalCommand(targetEnvironment Environment) []SyncCommand {
transferResource := m.GetTransferResource(targetEnvironment)

ret := []SyncCommand{}

substitutions := map[string]interface{}{
"transferResource": transferResource.Name,
}

for _, c := range m.Target.Commands {
ret = append(ret, generateSyncCommand(c, substitutions))
}

return ret
}

func (m CustomSyncRoot) GetFilesToCleanup(environment Environment) []string {
transferResource := m.GetTransferResource(environment)
return []string{
transferResource.Name,
}
}

func (m CustomSyncRoot) GetTransferResource(environment Environment) SyncerTransferResource {
return SyncerTransferResource{
Name: m.TransferResource,
IsDirectory: false}
}
Loading

0 comments on commit 6a83f78

Please sign in to comment.