diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index 24c1ab1c1cbf1..e857e6ac7fbfa 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -813,6 +813,28 @@ public void testMoveToInjectedStep() throws Exception { }); } + public void testCanStopILMWithPolicyUsingNonexistentPolicy() throws Exception { + createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), randomAlphaOfLengthBetween(5,15))); + + Request stopILMRequest = new Request("POST", "_ilm/stop"); + assertOK(client().performRequest(stopILMRequest)); + + Request statusRequest = new Request("GET", "_ilm/status"); + assertBusy(() -> { + Response statusResponse = client().performRequest(statusRequest); + assertOK(statusResponse); + Map statusResponseMap = entityAsMap(statusResponse); + String status = (String) statusResponseMap.get("operation_mode"); + assertEquals("STOPPED", status); + }); + + // Re-start ILM so that subsequent tests don't fail + Request startILMReqest = new Request("POST", "_ilm/start"); + assertOK(client().performRequest(startILMReqest)); + } + private void createFullPolicy(TimeValue hotTime) throws IOException { Map hotActions = new HashMap<>(); hotActions.put(SetPriorityAction.NAME, new SetPriorityAction(100)); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java index 378ca5dd49597..903e3811784c3 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java @@ -44,7 +44,7 @@ * A service which runs the {@link LifecyclePolicy}s associated with indexes. */ public class IndexLifecycleService extends AbstractComponent - implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable, LocalNodeMasterListener { + implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable, LocalNodeMasterListener { private static final Logger logger = LogManager.getLogger(IndexLifecycleService.class); private static final Set IGNORE_ACTIONS_MAINTENANCE_REQUESTED = Collections.singleton(ShrinkAction.NAME); private volatile boolean isMaster = false; @@ -111,18 +111,26 @@ public void onMaster() { IndexMetaData idxMeta = cursor.value; String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings()); if (Strings.isNullOrEmpty(policyName) == false) { - StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(LifecycleExecutionState.fromIndexMetadata(idxMeta)); - if (OperationMode.STOPPING == currentMode && - stepKey != null && - IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction()) == false) { - logger.info("skipping policy [{}] for index [{}]. stopping Index Lifecycle execution", - policyName, idxMeta.getIndex().getName()); - continue; + final LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(idxMeta); + StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState); + + if (OperationMode.STOPPING == currentMode) { + if (stepKey != null && IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction())) { + logger.info("waiting to stop ILM because index [{}] with policy [{}] is currently in action [{}]", + idxMeta.getIndex().getName(), policyName, stepKey.getAction()); + lifecycleRunner.maybeRunAsyncAction(clusterState, idxMeta, policyName, stepKey); + // ILM is trying to stop, but this index is in a Shrink action (or other dangerous action) so we can't stop + safeToStop = false; + } else { + logger.info("skipping policy execution for index [{}] with policy [{}] because ILM is stopping", + idxMeta.getIndex().getName(), policyName); + } + } else { + lifecycleRunner.maybeRunAsyncAction(clusterState, idxMeta, policyName, stepKey); } - lifecycleRunner.maybeRunAsyncAction(clusterState, idxMeta, policyName, stepKey); - safeToStop = false; // proven false! } } + if (safeToStop && OperationMode.STOPPING == currentMode) { submitOperationModeUpdate(OperationMode.STOPPED); } @@ -184,7 +192,7 @@ public void clusterChanged(ClusterChangedEvent event) { @Override public void applyClusterState(ClusterChangedEvent event) { if (event.localNodeMaster()) { // only act if we are master, otherwise - // keep idle until elected + // keep idle until elected if (event.state().metaData().custom(IndexLifecycleMetadata.TYPE) != null) { policyRegistry.update(event.state()); } @@ -237,21 +245,34 @@ void triggerPolicies(ClusterState clusterState, boolean fromClusterStateChange) IndexMetaData idxMeta = cursor.value; String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings()); if (Strings.isNullOrEmpty(policyName) == false) { - StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(LifecycleExecutionState.fromIndexMetadata(idxMeta)); - if (OperationMode.STOPPING == currentMode && stepKey != null - && IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction()) == false) { - logger.info("skipping policy [" + policyName + "] for index [" + idxMeta.getIndex().getName() - + "]. stopping Index Lifecycle execution"); - continue; - } - if (fromClusterStateChange) { - lifecycleRunner.runPolicyAfterStateChange(policyName, idxMeta); + final LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(idxMeta); + StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState); + + if (OperationMode.STOPPING == currentMode) { + if (stepKey != null && IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction())) { + logger.info("waiting to stop ILM because index [{}] with policy [{}] is currently in action [{}]", + idxMeta.getIndex().getName(), policyName, stepKey.getAction()); + if (fromClusterStateChange) { + lifecycleRunner.runPolicyAfterStateChange(policyName, idxMeta); + } else { + lifecycleRunner.runPeriodicStep(policyName, idxMeta); + } + // ILM is trying to stop, but this index is in a Shrink action (or other dangerous action) so we can't stop + safeToStop = false; + } else { + logger.info("skipping policy execution for index [{}] with policy [{}] because ILM is stopping", + idxMeta.getIndex().getName(), policyName); + } } else { - lifecycleRunner.runPeriodicStep(policyName, idxMeta); + if (fromClusterStateChange) { + lifecycleRunner.runPolicyAfterStateChange(policyName, idxMeta); + } else { + lifecycleRunner.runPeriodicStep(policyName, idxMeta); + } } - safeToStop = false; // proven false! } } + if (safeToStop && OperationMode.STOPPING == currentMode) { submitOperationModeUpdate(OperationMode.STOPPED); }