vuestorefront#308 Secure vsbridge reindexing

CHANGELOG.MD

@@ -1,6 +1,8 @@
 # Changelog
 
 ## Unreleased
+- Add optimization settings. You can now enable checking ES health before indexing and setting number_of_replicas=0
+and refresh_interval=-1 during indexing ([#308](https://github.com/DivanteLtd/magento2-vsbridge-indexer/issues/308))
 
 ## [1.19.0] (2020.06.23)
 

docs/configuration.md

@@ -98,6 +98,22 @@ Go to the new ‘Indexer’ section (Stores → Configuration → Vuestorefront
 
     ![](./images/config-cache.png) 
 
+1. ####Optimization Settings
+
+    **Check cluster health** → by default this option is disabled. Used to check cluster health before indexing.
+
+    **Check max bulk queue** → by default this option is disabled. Used to check if pending tasks + batch indexer size (VueStorefrontIndexer indices setting) are lower than max bulk queue size master node.
+
+    **Change number of replicas** → by default this option is disabled. Used to set number_of_replicas=0 during indexing.
+
+    **Default number of replicas** → number of replicas which is set after indexing.
+
+    **Change refresh interval** → by default this option is disabled. Used to set refresh_interval=-1 during indexing.
+
+    **Default value of refresh interval** → value of refresh interval which is set after indexing.
+
+    ![](./images/optimization-settings.png)
+
 1. ####Catalog Settings
 
     **Use Catalog Url Keys** → by default this option is disabled. Use Magento Url Key attribute for url_key and slug field (for products and categories). Url Keys have to be unique

src/module-vsbridge-indexer-core/Api/Client/ClientInterface.php

@@ -23,6 +23,22 @@ interface ClientInterface
      */
     public function bulk(array $bulkParams);
 
+    /**
+     * @param string $indexName
+     * @param int|string $value
+     *
+     * @return void
+     */
+    public function changeRefreshInterval(string $indexName, $value): void;
+
+    /**
+     * @param string $indexName
+     * @param int $value
+     *
+     * @return void
+     */
+    public function changeNumberOfReplicas(string $indexName, int $value): void;
+
     /**
      * @param string $indexName
      * @param array  $indexSettings
@@ -32,6 +48,14 @@ public function bulk(array $bulkParams);
      */
     public function createIndex(string $indexName, array $indexSettings);
 
+
+    /**
+     * Retrieve information about cluster health
+     *
+     * @return array
+     */
+    public function getClustersHealth(): array;
+
     /**
      * Retrieve the list of all index having a specified alias.
      *
@@ -41,6 +65,22 @@ public function createIndex(string $indexName, array $indexSettings);
      */
     public function getIndicesNameByAlias(string $indexAlias): array;
 
+    /**
+     * Retrieve information about index settings
+     *
+     * @param string $indexName
+     *
+     * @return array
+     */
+    public function getIndexSettings(string $indexName): array;
+
+    /**
+     * Retrieve max queue size for master node
+     *
+     * @return int
+     */
+    public function getMasterMaxQueueSize(): int;
+
     /**
      * @param array $aliasActions
      *

src/module-vsbridge-indexer-core/Api/IndexOperationInterface.php

@@ -88,4 +88,20 @@ public function createBulk();
      * @return int
      */
     public function getBatchIndexingSize();
+
+    /**
+     * @param int $storeId
+     * @param string $indexName
+     *
+     * @return void
+     */
+    public function optimizeEsIndexing($storeId, string $indexName);
+
+    /**
+     * @param int $storeId
+     * @param string $indexName
+     *
+     * @return void
+     */
+    public function cleanAfterOptimizeEsIndexing($storeId, string $indexName);
 }

src/module-vsbridge-indexer-core/Config/OptimizationSettings.php

@@ -0,0 +1,89 @@
+<?php
+/**
+ * Copyright Divante Sp. z o.o.
+ * See LICENSE_DIVANTE.txt for license details.
+ */
+
+namespace Divante\VsbridgeIndexerCore\Config;
+
+use Magento\Framework\App\Config\ScopeConfigInterface;
+
+class OptimizationSettings
+{
+    const OPTIMIZATION_SETTINGS_CONFIG_XML_PREFIX = 'vsbridge_indexer_settings/optimization_settings';
+
+    /**
+     * @var ScopeConfigInterface
+     */
+    private $scopeConfig;
+
+    /**
+     * ClientConfiguration constructor.
+     *
+     * @param ScopeConfigInterface $scopeConfig
+     */
+    public function __construct(ScopeConfigInterface $scopeConfig)
+    {
+        $this->scopeConfig = $scopeConfig;
+    }
+
+    /**
+     * @return bool
+     */
+    public function checkClusterHealth()
+    {
+        return (bool) $this->getConfigParam('cluster_health');
+    }
+
+    /**
+     * @return bool
+     */
+    public function checkMaxBulkQueueRequirement()
+    {
+        return (bool) $this->getConfigParam('max_bulk_queue_requirement');
+    }
+
+    /**
+     * @return bool
+     */
+    public function changeNumberOfReplicas()
+    {
+        return (bool) $this->getConfigParam('number_of_replicas');
+    }
+
+    /**
+     * @return string
+     */
+    public function getDefaultNumberOfReplicas()
+    {
+        return $this->getConfigParam('number_of_replicas_value');
+    }
+
+    /**
+     * @return bool
+     */
+    public function changeRefreshInterval()
+    {
+        return (bool) $this->getConfigParam('refresh_interval');
+    }
+
+    /**
+     * @return string
+     */
+    public function getDefaultRefreshInterval()
+    {
+        return $this->getConfigParam('refresh_interval_value') . 's';
+    }
+
+    /**
+     * @param string $configField
+     *
+     * @return string|null
+     */
+    private function getConfigParam(string $configField)
+    {
+        $path = self::OPTIMIZATION_SETTINGS_CONFIG_XML_PREFIX . '/' . $configField;
+
+        return $this->scopeConfig->getValue($path);
+    }
+}

src/module-vsbridge-indexer-core/Elasticsearch/Client.php

@@ -34,6 +34,28 @@ public function bulk(array $bulkParams)
         return $this->client->bulk($bulkParams);
     }
 
+    /**
+     * @param string $indexName
+     * @param int|string $value
+     *
+     * @return void
+     */
+    public function changeRefreshInterval(string $indexName, $value): void
+    {
+        $this->client->indices()->putSettings(['index' => $indexName, 'body' => ['refresh_interval' => $value]]);
+    }
+
+    /**
+     * @param string $indexName
+     * @param int $value
+     *
+     * @return void
+     */
+    public function changeNumberOfReplicas(string $indexName, int $value): void
+    {
+        $this->client->indices()->putSettings(['index' => $indexName, 'body' => ['number_of_replicas' => $value]]);
+    }
+
     /**
      * @param $indexName
      * @param array $indexSettings
@@ -48,6 +70,17 @@ public function createIndex(string $indexName, array $indexSettings)
         );
     }
 
+
+    /**
+     * Retrieve information about cluster health
+     *
+     * @return array
+     */
+    public function getClustersHealth(): array
+    {
+        return $this->client->cat()->health();
+    }
+
     /**
      * @param string $indexAlias
      *
@@ -65,6 +98,26 @@ public function getIndicesNameByAlias(string $indexAlias): array
         return array_keys($indices);
     }
 
+    /**
+     * @param string $indexName
+     *
+     * @return array
+     */
+    public function getIndexSettings(string $indexName): array
+    {
+        return $this->client->indices()->getSettings(['index' => $indexName]);
+    }
+
+    /**
+     * @return int
+     */
+    public function getMasterMaxQueueSize(): int
+    {
+        $master = $this->client->cat()->master();
+        $masterNode = $this->client->nodes()->info(['node_id' => $master[0]['id']]);
+        return $masterNode['nodes'][$master[0]['id']]['thread_pool']['search']['max_queue_size'] ?? 0;
+    }
+
     /**
      * @param array $aliasActions
      */

src/module-vsbridge-indexer-core/Exception/ConnectionUnhealthyException.php

@@ -0,0 +1,18 @@
+<?php
+/**
+ * Copyright Divante Sp. z o.o.
+ * See LICENSE_DIVANTE.txt for license details.
+ */
+
+namespace Divante\VsbridgeIndexerCore\Exception;
+
+use Magento\Framework\Exception\LocalizedException;
+
+/**
+ * Class ConnectionUnhealthyException
+ *
+ * @package Divante\VsbridgeIndexerCore\Exception
+ */
+class ConnectionUnhealthyException extends LocalizedException
+{
+}