test rex pull job against offline host

pytest_fixtures/core/broker.py

@@ -202,6 +202,13 @@ def module_capsule_configured_mqtt(module_capsule_configured):
     result = module_capsule_configured.execute('firewall-cmd --permanent --add-port="1883/tcp"')
     assert result.status == 0, 'Failed to open mqtt port on capsule'
     module_capsule_configured.execute('firewall-cmd --reload')
+    # lower the mqtt_resend_interval interval
+    # TODO use installer command instead once merged downstream
+    module_capsule_configured.execute(
+        "echo ':mqtt_resend_interval: 30' >> /etc/foreman-proxy/settings.d/remote_execution_ssh.yml"
+    )
+    result = module_capsule_configured.cli.Service.restart(options={'only': 'foreman-proxy'})
+    assert result.status == 0, 'foreman-proxy restart unsuccessful'
     yield module_capsule_configured
 
 

tests/foreman/cli/test_remoteexecution.py

@@ -1019,7 +1019,7 @@ def test_positive_run_job_on_host_converted_to_pull_provider(
         result = rhel_contenthost.execute('yum install -y katello-pull-transport-migrate')
         assert result.status == 0, 'Failed to install katello-pull-transport-migrate'
         # check mqtt client is running
-        result = rhel_contenthost.execute('yggdrasil status')
+        result = rhel_contenthost.execute('systemctl status yggdrasild')
         assert result.status == 0, f'Failed to start yggdrasil on client: {result.stderr}'
         result = rhel_contenthost.execute('systemctl status yggdrasild')
         assert result.status == 0, f'Failed to start yggdrasil on client: {result.stderr}'
@@ -1047,7 +1047,7 @@ def test_positive_run_job_on_host_converted_to_pull_provider(
         assert_job_invocation_result(invocation_command['id'], rhel_contenthost.hostname)
 
         # check katello-agent removal did not influence ygdrassil (SAT-1672)
-        result = rhel_contenthost.execute('yggdrasil status')
+        result = rhel_contenthost.execute('systemctl status yggdrasild')
         assert result.status == 0, f'Failed to start yggdrasil on client: {result.stderr}'
         result = rhel_contenthost.execute('systemctl status yggdrasild')
         assert result.status == 0, f'Failed to start yggdrasil on client: {result.stderr}'
@@ -1113,7 +1113,7 @@ def test_positive_run_job_on_host_registered_to_pull_provider(
 
         assert result.status == 0, f'Failed to register host: {result.stderr}'
         # check mqtt client is running
-        result = rhel_contenthost.execute('yggdrasil status')
+        result = rhel_contenthost.execute('systemctl status yggdrasild')
         assert result.status == 0, f'Failed to start yggdrasil on client: {result.stderr}'
         # run script provider rex command
         invocation_command = make_job_invocation(
@@ -1124,3 +1124,76 @@ def test_positive_run_job_on_host_registered_to_pull_provider(
             }
         )
         assert_job_invocation_result(invocation_command['id'], rhel_contenthost.hostname)
+
+    @pytest.mark.tier3
+    @pytest.mark.upgrade
+    @pytest.mark.no_containers
+    @pytest.mark.rhel_ver_match('[^6].*')
+    def test_positive_run_pull_job_on_offline_host(
+        self,
+        module_org,
+        module_target_sat,
+        smart_proxy_location,
+        module_ak_with_cv,
+        module_capsule_configured_mqtt,
+        rhel_contenthost,
+    ):
+        """Run pull-mqtt job against offline host
+
+        :id: c4914b78-6414-4a13-87b1-5b5cf01702a0
+
+        :expectedresults: Job is resumed when host comes back online
+
+        :CaseImportance: Critical
+
+        :parametrized: yes
+        """
+        client_repo = ohsnap.dogfood_repository(
+            settings.repos.ohsnap_repo_host,
+            product='client',
+            repo='client',
+            release='Client',
+            os_release=rhel_contenthost.os_version.major,
+        )
+        # Update module_capsule_configured_mqtt to include module_org/smart_proxy_location
+        module_target_sat.cli.Capsule.update(
+            {
+                'name': module_capsule_configured_mqtt.hostname,
+                'organization-ids': module_org.id,
+                'location-ids': smart_proxy_location.id,
+            }
+        )
+        result = rhel_contenthost.register(
+            module_org,
+            smart_proxy_location,
+            module_ak_with_cv.name,
+            target=module_capsule_configured_mqtt,
+            satellite=module_target_sat,
+            setup_remote_execution_pull=True,
+            repo=client_repo.baseurl,
+        )
+
+        assert result.status == 0, f'Failed to register host: {result.stderr}'
+        # check mqtt client is running
+        result = rhel_contenthost.execute('systemctl status yggdrasild')
+        assert result.status == 0, f'Failed to start yggdrasil on client: {result.stderr}'
+        # stop the client on host
+        result = rhel_contenthost.execute('systemctl stop yggdrasild')
+        assert result.status == 0, f'Failed to stop yggdrasil on client: {result.stderr}'
+        # run script provider rex command
+        invocation_command = make_job_invocation(
+            {
+                'job-template': 'Run Command - Script Default',
+                'inputs': 'command=ls',
+                'search-query': f'name ~ {rhel_contenthost.hostname}',
+                'async': True,
+            }
+        )
+        # assert the job is waiting to be picked up by client
+        assert_job_invocation_status(invocation_command['id'], rhel_contenthost.hostname, 'running')
+        # start client on host
+        result = rhel_contenthost.execute('systemctl start yggdrasild')
+        assert result.status == 0, f'Failed to start yggdrasil on client: {result.stderr}'
+        # wait twice the mqtt_resend_interval (set in module_capsule_configured_mqtt)
+        sleep(60)
+        assert_job_invocation_result(invocation_command['id'], rhel_contenthost.hostname)