added adminhtml configuration dropdown, update RecrawlService with dy…

…namic values

Helper/Data.php

@@ -128,6 +128,16 @@ class Data extends AbstractHelper
      */
     public const XML_PATH_INDEXER_MEMORY = 'nosto/flags/indexer_memory';
 
+    /**
+     * Product per request
+     */
+    public const XML_PATH_PRODUCT_PER_REQUEST = 'nosto/flags/product_per_request';
+
+    /**
+     * Request timeout
+     */
+    public const XML_PATH_REQUEST_TIMEOUT = 'nosto/flags/request_timeout';
+
     /**
      * Path to the configuration object that stores the preference for indexing disabled products
      */
@@ -441,6 +451,28 @@ public function getIndexerMemory(StoreInterface $store = null)
         return $this->getStoreConfig(self::XML_PATH_INDEXER_MEMORY, $store);
     }
 
+    /**
+     * Products per request
+     *
+     * @param StoreInterface|null $store the store model or null.
+     * @return int the configuration value
+     */
+    public function getProductsPerRequest(StoreInterface $store = null)
+    {
+        return $this->getStoreConfig(self::XML_PATH_PRODUCT_PER_REQUEST, $store);
+    }
+
+    /**
+     * Request timeout
+     *
+     * @param StoreInterface|null $store the store model or null.
+     * @return int the configuration value
+     */
+    public function getRequestTimeout(StoreInterface $store = null)
+    {
+        return $this->getStoreConfig(self::XML_PATH_REQUEST_TIMEOUT, $store);
+    }
+
     /**
      * Returns maximum percentage of PHP available memory that indexer should use
      *

Model/Config/Source/ProductsPerRequest.php

@@ -0,0 +1,59 @@
+<?php
+/**
+ * Copyright (c) 2020, Nosto Solutions Ltd
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its contributors
+ * may be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * @author Nosto Solutions Ltd <[email protected]>
+ * @copyright 2020 Nosto Solutions Ltd
+ * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause
+ *
+ */
+
+namespace Nosto\Tagging\Model\Config\Source;
+
+use Magento\Framework\Data\OptionSourceInterface;
+use Magento\Framework\Phrase;
+
+class ProductsPerRequest implements OptionSourceInterface
+{
+    /**
+     * Options getter
+     *
+     * @return array
+     */
+    public function toOptionArray()
+    {
+        return [
+            ['value' => 100, 'label' => new Phrase('100')],
+            ['value' => 200, 'label' => new Phrase('200')],
+            ['value' => 300, 'label' => new Phrase('300')],
+            ['value' => 400, 'label' => new Phrase('400')],
+            ['value' => 500, 'label' => new Phrase('500')],
+        ];
+    }
+}

Model/Config/Source/TimeBetweenBatchOfRequests.php

@@ -0,0 +1,59 @@
+<?php
+/**
+ * Copyright (c) 2020, Nosto Solutions Ltd
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its contributors
+ * may be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * @author Nosto Solutions Ltd <[email protected]>
+ * @copyright 2020 Nosto Solutions Ltd
+ * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause
+ *
+ */
+
+namespace Nosto\Tagging\Model\Config\Source;
+
+use Magento\Framework\Data\OptionSourceInterface;
+use Magento\Framework\Phrase;
+
+class TimeBetweenBatchOfRequests implements OptionSourceInterface
+{
+    /**
+     * Options getter
+     *
+     * @return array
+     */
+    public function toOptionArray()
+    {
+        return [
+            ['value' => 60, 'label' => new Phrase('60')],
+            ['value' => 120, 'label' => new Phrase('120')],
+            ['value' => 180, 'label' => new Phrase('180')],
+            ['value' => 240, 'label' => new Phrase('240')],
+            ['value' => 300, 'label' => new Phrase('300')],
+        ];
+    }
+}

Model/Service/Sync/Recrawl/RecrawlService.php

@@ -61,7 +61,6 @@ class RecrawlService extends AbstractService
 
     public const BENCHMARK_RECRAWL_NAME = 'nosto_product_recrawl';
     public const BENCHMARK_RECRAWL_BREAKPOINT = 1;
-    public const PRODUCT_RECRAWL_BATCH_SIZE = 100;
 
     /** @var CacheService */
     private CacheService $cacheService;
@@ -78,9 +77,6 @@ class RecrawlService extends AbstractService
     /** @var ProductRepository */
     private ProductRepository $productRepository;
 
-    /** @var int */
-    private int $recrawlBatchSize;
-
     /**
      * RecrawlService constructor.
      * @param CacheService $cacheService
@@ -90,7 +86,6 @@ class RecrawlService extends AbstractService
      * @param NostoHelperUrl $nostoHelperUrl
      * @param ProductRepository $productRepository
      * @param NostoLogger $logger
-     * @param int $recrawlBatchSize
      */
     public function __construct(
         CacheService $cacheService,
@@ -99,15 +94,13 @@ public function __construct(
         NostoHelperData $nostoHelperData,
         NostoHelperUrl $nostoHelperUrl,
         ProductRepository $productRepository,
-        NostoLogger $logger,
-        int $recrawlBatchSize = self::PRODUCT_RECRAWL_BATCH_SIZE
+        NostoLogger $logger
     ) {
         $this->cacheService = $cacheService;
         $this->productService = $productService;
         $this->nostoHelperAccount = $nostoHelperAccount;
         $this->nostoHelperUrl = $nostoHelperUrl;
         $this->productRepository = $productRepository;
-        $this->recrawlBatchSize = $recrawlBatchSize;
         parent::__construct($nostoHelperData, $nostoHelperAccount, $logger);
     }
 
@@ -132,8 +125,11 @@ public function recrawl(ProductCollection $collection, Store $store)
         $account = $this->nostoHelperAccount->findAccount($store);
         $this->startBenchmark(self::BENCHMARK_RECRAWL_NAME, self::BENCHMARK_RECRAWL_BREAKPOINT);
 
+        $timeBetweenBatchOfRequests = $this->getDataHelper()->getRequestTimeout();
+        $productsPerRequest = $this->getDataHelper()->getProductsPerRequest();
+
         $index = 0;
-        $collection->setPageSize($this->recrawlBatchSize);
+        $collection->setPageSize($productsPerRequest);
         $iterator = new PagingIterator($collection);
 
         /** @var ProductCollection $page */
@@ -144,8 +140,8 @@ public function recrawl(ProductCollection $collection, Store $store)
             $op->setResponseTimeout(60);
             $products = $this->productRepository->getByIds(
                 $page->getAllIds(
-                    $this->recrawlBatchSize,
-                    ($iterator->getCurrentPageNumber() - 1) * $this->recrawlBatchSize
+                    $productsPerRequest,
+                    ($iterator->getCurrentPageNumber() - 1) * $productsPerRequest
                 )
             );
             /** @var Product $product */
@@ -167,13 +163,14 @@ public function recrawl(ProductCollection $collection, Store $store)
             $this->logDebugWithStore(
                 sprintf(
                     'Upserting batch of %d (%s) - API timeout is set to %d seconds',
-                    $this->recrawlBatchSize,
+                    $productsPerRequest,
                     implode(',', $productIdsInBatch),
                     60
                 ),
                 $store
             );
             $op->requestRecrawl();
+            sleep($timeBetweenBatchOfRequests);
         }
         $this->logBenchmarkSummary(self::BENCHMARK_RECRAWL_NAME, $store, $this);
     }

etc/adminhtml/system.xml

@@ -191,6 +191,22 @@
                     </comment>
                     <source_model>Nosto\Tagging\Model\Config\Source\Memory</source_model>
                 </field>
+                <field id="product_per_request" translate="label comment" type="select" sortOrder="110"
+                       showInDefault="1" showInWebsite="1" showInStore="1">
+                    <label>Light Recrawl Indexer - Products per Request</label>
+                    <comment>
+                        <![CDATA[Set the number of product IDs to be sent to Nosto in each request. A lower number can prevent overwhelming Nosto during updates.]]>
+                    </comment>
+                    <source_model>Nosto\Tagging\Model\Config\Source\ProductsPerRequest</source_model>
+                </field>
+                <field id="time_between_batch_of_requests" translate="label comment" type="select" sortOrder="110"
+                       showInDefault="1" showInWebsite="1" showInStore="1">
+                    <label>Light Recrawl Indexer - Time between batch of requests (seconds)</label>
+                    <comment>
+                        <![CDATA[Define the time between batch of requests to Nosto. This helps manage traffic and prevents potential spikes that could affect performance.]]>
+                    </comment>
+                    <source_model>Nosto\Tagging\Model\Config\Source\TimeBetweenBatchOfRequests</source_model>
+                </field>
                 <field id="indexer_disabled_products" translate="label comment" type="select" sortOrder="120"
                        showInDefault="1" showInWebsite="1" showInStore="1">
                     <label>Index disabled products</label>

etc/config.xml

@@ -56,6 +56,8 @@
                 <low_stock_indication>0</low_stock_indication>
                 <send_customer_data>1</send_customer_data>
                 <indexer_memory>50</indexer_memory>
+                <product_per_request>100</product_per_request>
+                <time_between_batch_of_requests>60</time_between_batch_of_requests>
                 <indexer_disabled_products>0</indexer_disabled_products>
                 <tag_date_published>0</tag_date_published>
                 <track_multi_channel_orders>1</track_multi_channel_orders>

etc/indexer.xml

@@ -40,7 +40,7 @@
     </indexer>
 
     <indexer id="nosto_index_light_product" view_id="nosto_index_light_product" class="Nosto\Tagging\Model\Indexer\LightProductIndexer">
-        <title>Nosto Product Indexer</title>
+        <title>Nosto Product Light Indexer</title>
         <description>Populates message queue with product ids to be sent to Nosto</description>
     </indexer>
 </config>