(PE-39352) Update backup/restore plans for hac database

PE 2023.7.0 added the host-action-collector service and the
backup/restore plans need to be made aware of the pe-hac database.

This patch adds a version test, best on the addition of pe_version check
of /opt/puppetlabs/server/pe_build in the peadm::get_peadm_config task.

If >= 2023.7, then pe-hac is added to the defaults returned by the
various recover_opts functions that define which resource backup/restore
should wrangle.

The patch does not attempt to version test the user's custom override
hash of selections, on the assumption that they know, or should be
allowed to specify whatever is needed.

If pe_version is not found (damaged cluster, missing from
peadm_config.json), the patch defaults to assuming earliest version,
which will skip pe-hac. This favors the plans running without error at
the cost of possibly missing pe-hac in a newer installation.
Alternately, we could reverse this and incur failures in damaged older
clusters requiring use of $custom to work around.

Finally there is an edge case it does not address which is the case of
newer pe versions with pe-hac restoring tarballs that were intended to
include all databases, but which were created with peadm predating this
patch. These will fail the pg-restore pe-hac commands, since pe-hac
won't be in the backup, and again require use of $custom to work around.

functions/amend_recovery_defaults_by_pe_version.pp

@@ -0,0 +1,18 @@
+function peadm::amend_recovery_defaults_by_pe_version (
+  Hash              $base_opts,
+  Peadm::Pe_version $pe_version,
+  Boolean           $opt_value,
+) {
+  # work around puppet-lint check_unquoted_string_in_case
+  $semverrange = SemVerRange('>= 2023.7')
+  case $pe_version {
+    $semverrange: {
+      $base_opts + {
+        'hac' => $opt_value,
+      }
+    }
+    default: {
+      $base_opts
+    }
+  }
+}

functions/migration_opts_default.pp

@@ -1,5 +1,7 @@
-function peadm::migration_opts_default () {
-  {
+function peadm::migration_opts_default (
+  Peadm::Pe_version $pe_version,
+) {
+  $base_opts = {
     'activity'     => true,
     'ca'           => true,
     'classifier'   => true,
@@ -9,4 +11,5 @@ function peadm::migration_opts_default () {
     'puppetdb'     => true,
     'rbac'         => true,
   }
+  peadm::amend_recovery_defaults_by_pe_version($base_opts, $pe_version, true)
 }

functions/recovery_opts_all.pp

@@ -1,5 +1,7 @@
-function peadm::recovery_opts_all () {
-  {
+function peadm::recovery_opts_all (
+  Peadm::Pe_version $pe_version,
+) {
+  $base_opts = {
     'activity'     => true,
     'ca'           => true,
     'classifier'   => true,
@@ -9,4 +11,5 @@ function peadm::recovery_opts_all () {
     'puppetdb'     => true,
     'rbac'         => true,
   }
+  peadm::amend_recovery_defaults_by_pe_version($base_opts, $pe_version, true)
 }

functions/recovery_opts_default.pp

@@ -1,5 +1,7 @@
-function peadm::recovery_opts_default () {
-  {
+function peadm::recovery_opts_default (
+  Peadm::Pe_version $pe_version,
+) {
+  $base_opts = {
     'activity'     => false,
     'ca'           => true,
     'classifier'   => false,
@@ -9,4 +11,5 @@ function peadm::recovery_opts_default () {
     'puppetdb'     => true,
     'rbac'         => false,
   }
+  peadm::amend_recovery_defaults_by_pe_version($base_opts, $pe_version, false)
 }

functions/validated_pe_version_for_backup_restore.pp

@@ -0,0 +1,22 @@
+# Verify that *pe_version* string is a valid SemVer.
+# If not, warn, and return "0.0.0" as a permissive default.
+function peadm::validated_pe_version_for_backup_restore(
+  Optional[String] $pe_version,
+) {
+  # work around puppet-lint check_unquoted_string_in_case
+  $semverrange = SemVerRange('>=0.0.0')
+  case $pe_version {
+    # Validate that the value is a SemVer value.
+    $semverrange: {
+      $pe_version
+    }
+    default: {
+      $msg = @("WARN")
+        WARNING: Retrieved a missing or unparseable PE version of '${pe_version}'.
+        The host_action_collector database will be skipped from defaults.
+        |-WARN
+      out::message($msg)
+      '0.0.0'
+    }
+  }
+}

plans/backup.pp

@@ -35,10 +35,12 @@
     getvar('cluster.params.compiler_hosts'),
   )
 
+  $pe_version = peadm::validated_pe_version_for_backup_restore(getvar('cluster.pe_version'))
+
   $recovery_opts = $backup_type? {
-    'recovery'  => peadm::recovery_opts_default(),
-    'migration' => peadm::migration_opts_default(),
-    'custom'    => peadm::recovery_opts_all() + $backup,
+    'recovery'  => peadm::recovery_opts_default($pe_version),
+    'migration' => peadm::migration_opts_default($pe_version),
+    'custom'    => peadm::recovery_opts_all($pe_version) + $backup,
   }
 
   $timestamp = Timestamp.new().strftime('%Y-%m-%dT%H%M%SZ')
@@ -55,6 +57,8 @@
     'activity'     => $primary_target,
     'rbac'         => $primary_target,
     'puppetdb'     => $puppetdb_postgresql_target,
+    # (host-action-collector db will be filtered for pe version by recovery_opts)
+    'hac'          => $primary_target,
   }.filter |$key,$_| {
     $recovery_opts[$key] == true
   }

plans/restore.pp

@@ -34,7 +34,7 @@
   # try to load the cluster configuration by running peadm::get_peadm_config, but allow for errors to happen
   $_cluster = run_task('peadm::get_peadm_config', $targets, { '_catch_errors' => true }).first.value
 
-  if $_cluster == undef or getvar('_cluster.params') == undef {
+  if $_cluster == undef or getvar('_cluster.params') == undef or getvar('_cluster.pe_version') == undef {
     # failed to get cluster config, load from backup
     out::message('Failed to get cluster configuration, loading from backup...')
     $result = download_file("${recovery_directory}/peadm/peadm_config.json", 'peadm_config.json', $targets).first.value
@@ -59,11 +59,13 @@
     getvar('cluster.params.compiler_hosts'),
   )
 
+  $pe_version = peadm::validated_pe_version_for_backup_restore(getvar('cluster.pe_version'))
+
   $recovery_opts = $restore_type? {
-    'recovery'     => peadm::recovery_opts_default(),
+    'recovery'     => peadm::recovery_opts_default($pe_version),
     'recovery-db'  => { 'puppetdb' => true, },
-    'migration'    => peadm::migration_opts_default(),
-    'custom'       => peadm::recovery_opts_all() + $restore,
+    'migration'    => peadm::migration_opts_default($pe_version),
+    'custom'       => peadm::recovery_opts_all($pe_version) + $restore,
   }
 
   $primary_target   = peadm::get_targets(getvar('cluster.params.primary_host'), 1)
@@ -97,6 +99,8 @@
     'activity'     => [$primary_target],
     'rbac'         => [$primary_target],
     'puppetdb'     => $puppetdb_postgresql_targets,
+    # (host-action-collector db will be filtered for pe version by recovery_opts)
+    'hac'          => $primary_target,
   }.filter |$key,$_| {
     $recovery_opts[$key] == true
   }
@@ -203,7 +207,7 @@
   if getvar('recovery_opts.orchestrator') {
     out::message('# Restoring orchestrator secret keys')
     run_command(@("CMD"/L), $primary_target)
-      cp -rp ${shellquote($recovery_directory)}/orchestrator/secrets/* /etc/puppetlabs/orchestration-services/conf.d/secrets/ 
+      cp -rp ${shellquote($recovery_directory)}/orchestrator/secrets/* /etc/puppetlabs/orchestration-services/conf.d/secrets/
       | CMD
   }
 # lint:endignore

spec/fixtures/peadm_config.json

@@ -1,3 +1,4 @@
 {
+  "pe_version" : "2023.7.0",
   "params": { "primary_host": "primary", "primary_postgresql_host": "postgres" }
 }

spec/fixtures/peadm_config.no_pe_version.json

@@ -0,0 +1,3 @@
+{
+  "params": { "primary_host": "primary", "primary_postgresql_host": "postgres" }
+}

spec/functions/amend_recovery_defaults_by_pe_version_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe 'peadm::amend_recovery_defaults_by_pe_version' do
+  it 'just returns the base opts if version < 2023.7' do
+    is_expected.to run.with_params({}, '2023.6.0', true).and_return({})
+  end
+
+  it 'adds hac if version >= 2023.7' do
+    is_expected.to run.with_params({}, '2023.7.0', true).and_return({ 'hac' => true })
+  end
+
+  it 'adds hac false based on opt_value' do
+    is_expected.to run.with_params({}, '2023.7.0', false).and_return({ 'hac' => false })
+  end
+end

spec/functions/migration_opts_default_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+describe 'peadm::migration_opts_default' do
+  it 'returns pre 2023.7 defaults' do
+    is_expected.to run.with_params('2023.6.0').and_return(
+      {
+        'activity'     => true,
+        'ca'           => true,
+        'classifier'   => true,
+        'code'         => false,
+        'config'       => false,
+        'orchestrator' => true,
+        'puppetdb'     => true,
+        'rbac'         => true,
+      },
+    )
+  end
+
+  it 'returns 2023.7+ defaults with hac' do
+    is_expected.to run.with_params('2023.7.0').and_return(
+      {
+        'activity'     => true,
+        'ca'           => true,
+        'classifier'   => true,
+        'code'         => false,
+        'config'       => false,
+        'orchestrator' => true,
+        'puppetdb'     => true,
+        'rbac'         => true,
+        'hac'          => true,
+      },
+    )
+  end
+end

spec/functions/recovery_opts_all_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+describe 'peadm::recovery_opts_all' do
+  it 'returns pre 2023.7 defaults' do
+    is_expected.to run.with_params('2023.6.0').and_return(
+      {
+        'activity'     => true,
+        'ca'           => true,
+        'classifier'   => true,
+        'code'         => true,
+        'config'       => true,
+        'orchestrator' => true,
+        'puppetdb'     => true,
+        'rbac'         => true,
+      },
+    )
+  end
+
+  it 'returns 2023.7+ defaults with hac' do
+    is_expected.to run.with_params('2023.7.0').and_return(
+      {
+        'activity'     => true,
+        'ca'           => true,
+        'classifier'   => true,
+        'code'         => true,
+        'config'       => true,
+        'orchestrator' => true,
+        'puppetdb'     => true,
+        'rbac'         => true,
+        'hac'          => true,
+      },
+    )
+  end
+end

spec/functions/recovery_opts_default_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+describe 'peadm::recovery_opts_default' do
+  it 'returns pre 2023.7 defaults' do
+    is_expected.to run.with_params('2023.6.0').and_return(
+      {
+        'activity'     => false,
+        'ca'           => true,
+        'classifier'   => false,
+        'code'         => true,
+        'config'       => true,
+        'orchestrator' => false,
+        'puppetdb'     => true,
+        'rbac'         => false,
+      },
+    )
+  end
+
+  it 'returns 2023.7+ defaults with hac' do
+    is_expected.to run.with_params('2023.7.0').and_return(
+      {
+        'activity'     => false,
+        'ca'           => true,
+        'classifier'   => false,
+        'code'         => true,
+        'config'       => true,
+        'orchestrator' => false,
+        'puppetdb'     => true,
+        'rbac'         => false,
+        'hac'          => false,
+      },
+    )
+  end
+end