Skip to content

Commit

Permalink
k8sd cluster-recover: add non-interactive mode
Browse files Browse the repository at this point in the history 
At the moment, the "k8sd cluster-recover" displays interactive
prompts and text editors that assist the user in updating the dqlite
configuration.

We need to be able to run the command non-interactively in order
to automate the cluster recovery procedure.

This change adds a "--non-interactive" flag. If set, we'll no longer
show confirmation prompts and we'll assume that the configuration
files have already been updated, proceeding with the dqlite recovery.
  • Loading branch information
@petrutlucian94
petrutlucian94 committed Sep 11, 2024
1 parent 898cce0 commit 922f2a4
Show file tree
Hide file tree
Showing 2 changed files with 123 additions and 92 deletions.
8 changes: 8 additions & 0 deletions docs/src/snap/howto/restore-quorum.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,14 @@ needs to be skipped, use the ``--skip-k8sd`` or ``--skip-k8s-dqlite`` flags.
This can be useful when using an external Etcd database.
```

```{note}
Non-interactive mode can be requested using the ``--non-interactive`` flag.
In this case, no interactive prompts or text editors will be displayed and
the command will assume that the configuration files have already been updated.
This allows automating the recovery procedure.
```

Once the "cluster-recover" command completes, restart the k8s services on the node:

Check failure on line 105 in docs/src/snap/howto/restore-quorum.md

View workflow job for this annotation

GitHub Actions / markdown-lint

Line length 

docs/src/snap/howto/restore-quorum.md:105:81 MD013/line-length Line length [Expected: 80; Actual: 83] https://github.com/DavidAnson/markdownlint/blob/v0.34.0/doc/md013.md

```
Expand Down
207 changes: 115 additions & 92 deletions src/k8s/cmd/k8sd/k8sd_cluster_recover.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ const yamlHelperCommentFooter = "# ------- everything below will be written ----

var clusterRecoverOpts struct {
K8sDqliteStateDir string
NonInteractive bool
SkipK8sd bool
SkipK8sDqlite bool
}
Expand Down Expand Up @@ -145,6 +146,8 @@ func newClusterRecoverCmd() *cobra.Command {

cmd.Flags().StringVar(&clusterRecoverOpts.K8sDqliteStateDir, "k8s-dqlite-state-dir",
"", "k8s-dqlite datastore location")
cmd.Flags().BoolVar(&clusterRecoverOpts.NonInteractive, "non-interactive",
false, "disable interactive prompts, assume that the configs have been updated")
cmd.Flags().BoolVar(&clusterRecoverOpts.SkipK8sd, "skip-k8sd",
false, "skip k8sd recovery")
cmd.Flags().BoolVar(&clusterRecoverOpts.SkipK8sDqlite, "skip-k8s-dqlite",
Expand All @@ -171,8 +174,8 @@ func recoveryCmdPrechecks(ctx context.Context) error {

log.V(1).Info("Running prechecks.")

if !termios.IsTerminal(unix.Stdin) {
return fmt.Errorf("this command is meant to be run in an interactive terminal")
if !termios.IsTerminal(unix.Stdin) && !clusterRecoverOpts.NonInteractive {
return fmt.Errorf("interactive mode requested in a non-interactive terminal")
}

if clusterRecoverOpts.K8sDqliteStateDir == "" {
Expand All @@ -185,18 +188,20 @@ func recoveryCmdPrechecks(ctx context.Context) error {
reader := bufio.NewReader(os.Stdin)
fmt.Print(recoveryConfirmation)

input, err := reader.ReadString('\n')
if err != nil {
return fmt.Errorf("couldn't read user input, error: %w", err)
}
input = strings.TrimSuffix(input, "\n")
if !clusterRecoverOpts.NonInteractive {
input, err := reader.ReadString('\n')
if err != nil {
return fmt.Errorf("couldn't read user input, error: %w", err)
}
input = strings.TrimSuffix(input, "\n")

if strings.ToLower(input) != "yes" {
return fmt.Errorf("cluster edit aborted; no changes made")
if strings.ToLower(input) != "yes" {
return fmt.Errorf("cluster edit aborted; no changes made")
}
}

if !clusterRecoverOpts.SkipK8sDqlite {
if err = ensureK8sDqliteMembersStopped(ctx); err != nil {
if err := ensureK8sDqliteMembersStopped(ctx); err != nil {
return err
}
}
Expand Down Expand Up @@ -376,59 +381,64 @@ func recoverK8sd() (string, error) {
clusterYamlPath := path.Join(m.FileSystem.DatabaseDir, "cluster.yaml")
clusterYamlCommentHeader := fmt.Sprintf("# K8sd cluster configuration\n# (based on the trust store and %s)\n", clusterYamlPath)

clusterYamlContent, err := yamlEditorGuide(
"",
false,
slices.Concat(
[]byte(clusterYamlCommentHeader),
[]byte("#\n"),
[]byte(clusterK8sdYamlRecoveryComment),
[]byte(yamlHelperCommentFooter),
[]byte("\n"),
oldMembersYaml,
),
false,
)
if err != nil {
return "", fmt.Errorf("interactive text editor failed, error: %w", err)
}
clusterYamlContent := oldMembersYaml
if clusterRecoverOpts.NonInteractive {
// Interactive mode requested (default).
// Assist the user in configuring dqlite.
clusterYamlContent, err = yamlEditorGuide(
"",
false,
slices.Concat(
[]byte(clusterYamlCommentHeader),
[]byte("#\n"),
[]byte(clusterK8sdYamlRecoveryComment),
[]byte(yamlHelperCommentFooter),
[]byte("\n"),
oldMembersYaml,
),
false,
)
if err != nil {
return "", fmt.Errorf("interactive text editor failed, error: %w", err)
}

infoYamlPath := path.Join(m.FileSystem.DatabaseDir, "info.yaml")
infoYamlCommentHeader := fmt.Sprintf("# K8sd info.yaml\n# (%s)\n", infoYamlPath)
_, err = yamlEditorGuide(
infoYamlPath,
true,
slices.Concat(
[]byte(infoYamlCommentHeader),
[]byte("#\n"),
[]byte(infoYamlRecoveryComment),
utils.YamlCommentLines(clusterYamlContent),
[]byte("\n"),
[]byte(yamlHelperCommentFooter),
),
true,
)
if err != nil {
return "", fmt.Errorf("interactive text editor failed, error: %w", err)
}
infoYamlPath := path.Join(m.FileSystem.DatabaseDir, "info.yaml")
infoYamlCommentHeader := fmt.Sprintf("# K8sd info.yaml\n# (%s)\n", infoYamlPath)
_, err = yamlEditorGuide(
infoYamlPath,
true,
slices.Concat(
[]byte(infoYamlCommentHeader),
[]byte("#\n"),
[]byte(infoYamlRecoveryComment),
utils.YamlCommentLines(clusterYamlContent),
[]byte("\n"),
[]byte(yamlHelperCommentFooter),
),
true,
)
if err != nil {
return "", fmt.Errorf("interactive text editor failed, error: %w", err)
}

daemonYamlPath := path.Join(m.FileSystem.StateDir, "daemon.yaml")
daemonYamlCommentHeader := fmt.Sprintf("# K8sd daemon.yaml\n# (%s)\n", daemonYamlPath)
_, err = yamlEditorGuide(
daemonYamlPath,
true,
slices.Concat(
[]byte(daemonYamlCommentHeader),
[]byte("#\n"),
[]byte(daemonYamlRecoveryComment),
utils.YamlCommentLines(clusterYamlContent),
[]byte("\n"),
[]byte(yamlHelperCommentFooter),
),
true,
)
if err != nil {
return "", fmt.Errorf("interactive text editor failed, error: %w", err)
daemonYamlPath := path.Join(m.FileSystem.StateDir, "daemon.yaml")
daemonYamlCommentHeader := fmt.Sprintf("# K8sd daemon.yaml\n# (%s)\n", daemonYamlPath)
_, err = yamlEditorGuide(
daemonYamlPath,
true,
slices.Concat(
[]byte(daemonYamlCommentHeader),
[]byte("#\n"),
[]byte(daemonYamlRecoveryComment),
utils.YamlCommentLines(clusterYamlContent),
[]byte("\n"),
[]byte(yamlHelperCommentFooter),
),
true,
)
if err != nil {
return "", fmt.Errorf("interactive text editor failed, error: %w", err)
}
}

newMembers := []cluster.DqliteMember{}
Expand Down Expand Up @@ -465,40 +475,53 @@ func recoverK8sd() (string, error) {
func recoverK8sDqlite() (string, string, error) {
k8sDqliteStateDir := clusterRecoverOpts.K8sDqliteStateDir

var err error
clusterYamlContent := []byte{}
clusterYamlPath := path.Join(k8sDqliteStateDir, "cluster.yaml")
clusterYamlCommentHeader := fmt.Sprintf("# k8s-dqlite cluster configuration\n# (%s)\n", clusterYamlPath)
clusterYamlContent, err := yamlEditorGuide(
clusterYamlPath,
true,
slices.Concat(
[]byte(clusterYamlCommentHeader),
[]byte("#\n"),
[]byte(clusterK8sDqliteRecoveryComment),
[]byte(yamlHelperCommentFooter),
),
true,
)
if err != nil {
return "", "", fmt.Errorf("interactive text editor failed, error: %w", err)
}

infoYamlPath := path.Join(k8sDqliteStateDir, "info.yaml")
infoYamlCommentHeader := fmt.Sprintf("# k8s-dqlite info.yaml\n# (%s)\n", infoYamlPath)
_, err = yamlEditorGuide(
infoYamlPath,
true,
slices.Concat(
[]byte(infoYamlCommentHeader),
[]byte("#\n"),
[]byte(infoYamlRecoveryComment),
utils.YamlCommentLines(clusterYamlContent),
[]byte("\n"),
[]byte(yamlHelperCommentFooter),
),
true,
)
if err != nil {
return "", "", fmt.Errorf("interactive text editor failed, error: %w", err)
if clusterRecoverOpts.NonInteractive {
clusterYamlContent, err = os.ReadFile(clusterYamlPath)
if err != nil {
return "", "", fmt.Errorf(
"could not read k8s-dqlite cluster.yaml, error: %w", err)
}
} else {
// Interactive mode requested (default).
// Assist the user in configuring dqlite.
clusterYamlContent, err = yamlEditorGuide(
clusterYamlPath,
true,
slices.Concat(
[]byte(clusterYamlCommentHeader),
[]byte("#\n"),
[]byte(clusterK8sDqliteRecoveryComment),
[]byte(yamlHelperCommentFooter),
),
true,
)
if err != nil {
return "", "", fmt.Errorf("interactive text editor failed, error: %w", err)
}

infoYamlPath := path.Join(k8sDqliteStateDir, "info.yaml")
infoYamlCommentHeader := fmt.Sprintf("# k8s-dqlite info.yaml\n# (%s)\n", infoYamlPath)
_, err = yamlEditorGuide(
infoYamlPath,
true,
slices.Concat(
[]byte(infoYamlCommentHeader),
[]byte("#\n"),
[]byte(infoYamlRecoveryComment),
utils.YamlCommentLines(clusterYamlContent),
[]byte("\n"),
[]byte(yamlHelperCommentFooter),
),
true,
)
if err != nil {
return "", "", fmt.Errorf("interactive text editor failed, error: %w", err)
}
}

newMembers := []dqlite.NodeInfo{}
Expand Down

0 comments on commit 922f2a4

Please sign in to comment.