From 063b516e57e431940a13d2dca7f9b78747661eef Mon Sep 17 00:00:00 2001 From: Cid Lopes Date: Tue, 16 Jul 2024 20:16:06 +0300 Subject: [PATCH 1/5] Init recrawl light indexer --- Helper/Data.php | 16 ++ Model/Indexer/LightProductIndexer.php | 155 +++++++++++++++ Model/Product/Builder.php | 41 ++++ .../Service/Product/DefaultProductService.php | 29 +++ Model/Service/Sync/Recrawl/RecrawlService.php | 181 ++++++++++++++++++ etc/di.xml | 10 + etc/indexer.xml | 5 + etc/mview.xml | 30 +++ 8 files changed, 467 insertions(+) create mode 100644 Model/Indexer/LightProductIndexer.php create mode 100644 Model/Service/Sync/Recrawl/RecrawlService.php diff --git a/Helper/Data.php b/Helper/Data.php index 3b1895106..4e62e7706 100644 --- a/Helper/Data.php +++ b/Helper/Data.php @@ -168,6 +168,11 @@ class Data extends AbstractHelper */ public const XML_PATH_MULTI_CURRENCY = 'nosto/multicurrency/method'; + /** + * Path to the configuration object for light indexer + */ + public const XML_PATH_USE_LIGHT_INDEXER = 'nosto/flags/use_light_indexer'; + /** * @var string Nosto customer reference attribute name */ @@ -504,6 +509,17 @@ public function getMultiCurrencyMethod(StoreInterface $store = null) return $this->getStoreConfig(self::XML_PATH_MULTI_CURRENCY, $store); } + /** + * Returns if the light indexer should be used + * + * @param StoreInterface|null $store the store model or null. + * @return string the configuration value + */ + public function getUseLightIndexer(StoreInterface $store = null) + { + return $this->getStoreConfig(self::XML_PATH_USE_LIGHT_INDEXER, $store); + } + /** * Saves the multi currency setup value / multi currency method * diff --git a/Model/Indexer/LightProductIndexer.php b/Model/Indexer/LightProductIndexer.php new file mode 100644 index 000000000..c92ef4238 --- /dev/null +++ b/Model/Indexer/LightProductIndexer.php @@ -0,0 +1,155 @@ + + * @copyright 2020 Nosto Solutions Ltd + * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause + * + */ + +namespace Nosto\Tagging\Model\Indexer; + +use Exception; +use Magento\Indexer\Model\ProcessManager; +use Magento\Store\Model\App\Emulation; +use Magento\Store\Model\Store; +use Nosto\NostoException; +use Nosto\Tagging\Helper\Scope as NostoHelperScope; +use Nosto\Tagging\Logger\Logger as NostoLogger; +use Nosto\Tagging\Model\Indexer\Dimensions\Product\ModeSwitcher as ProductModeSwitcher; +use Nosto\Tagging\Model\Indexer\Dimensions\ModeSwitcherInterface; +use Nosto\Tagging\Model\Indexer\Dimensions\StoreDimensionProvider; +use Nosto\Tagging\Model\ResourceModel\Magento\Product\Collection as ProductCollection; +use Nosto\Tagging\Model\ResourceModel\Magento\Product\CollectionBuilder; +use Nosto\Tagging\Model\Service\Indexer\IndexerStatusServiceInterface; +use Symfony\Component\Console\Input\InputInterface; +use Nosto\Tagging\Model\Service\Sync\Recrawl\RecrawlService; + +/** + * Class ProductIndexer + * Fetches product ID's from CL tables and create entries in the message queue + */ +class LightProductIndexer extends AbstractIndexer +{ + public const INDEXER_ID = 'nosto_index_light_product'; + + /** @var CollectionBuilder */ + private CollectionBuilder $productCollectionBuilder; + + /** @var ProductModeSwitcher */ + private ProductModeSwitcher $modeSwitcher; + + /** @var RecrawlService */ + private RecrawlService $recrawlService; + + /** + * Invalidate constructor. + * @param NostoHelperScope $nostoHelperScope + * @param NostoLogger $logger + * @param CollectionBuilder $productCollectionBuilder + * @param ProductModeSwitcher $modeSwitcher + * @param StoreDimensionProvider $dimensionProvider + * @param Emulation $storeEmulation + * @param ProcessManager $processManager + * @param InputInterface $input + * @param IndexerStatusServiceInterface $indexerStatusService + * @param RecrawlService $recrawlService + */ + public function __construct( + NostoHelperScope $nostoHelperScope, + NostoLogger $logger, + CollectionBuilder $productCollectionBuilder, + ProductModeSwitcher $modeSwitcher, + StoreDimensionProvider $dimensionProvider, + Emulation $storeEmulation, + ProcessManager $processManager, + InputInterface $input, + IndexerStatusServiceInterface $indexerStatusService, + RecrawlService $recrawlService + ) { + $this->productCollectionBuilder = $productCollectionBuilder; + $this->modeSwitcher = $modeSwitcher; + $this->recrawlService = $recrawlService; + parent::__construct( + $nostoHelperScope, + $logger, + $dimensionProvider, + $storeEmulation, + $input, + $indexerStatusService, + $processManager + ); + } + + /** + * @inheritDoc + */ + public function getModeSwitcher(): ModeSwitcherInterface + { + return $this->modeSwitcher; + } + + /** + * @inheritDoc + * @throws NostoException + * @throws Exception + */ + public function doIndex(Store $store, array $ids = []) + { +// if ($this->nostoHelperData->getUseLightIndexer($store)) { + $collection = $this->getCollection($store, $ids); + $this->recrawlService->recrawl($collection, $store); +// } + } + + /** + * @inheritDoc + */ + public function getIndexerId(): string + { + return self::INDEXER_ID; + } + + /** + * @param Store $store + * @param array $ids + * @return ProductCollection + */ + public function getCollection(Store $store, array $ids = []): ProductCollection + { + $this->productCollectionBuilder->initDefault($store); + if (!empty($ids)) { + $this->productCollectionBuilder->withIds($ids); + } else { + $this->productCollectionBuilder->withDefaultVisibility($store); + } + return $this->productCollectionBuilder->build(); + } +} diff --git a/Model/Product/Builder.php b/Model/Product/Builder.php index a9cae9432..0b0c58b52 100644 --- a/Model/Product/Builder.php +++ b/Model/Product/Builder.php @@ -343,6 +343,47 @@ public function build( return $nostoProduct; } + /** + * @param Product $product + * @param Store $store + * @return NostoProduct + * @throws FilteredProductException + * @throws NonBuildableProductException + */ + public function buildLightProduct( + Product $product, + Store $store + ) { + $nostoProduct = new NostoProduct(); + $modelFilter = new ModelFilter(); + $this->eventManager->dispatch( + 'nosto_product_load_before', + ['product' => $nostoProduct, 'magentoProduct' => $product, 'modelFilter' => $modelFilter] + ); + if (!$modelFilter->isValid()) { + throw new FilteredProductException( + sprintf( + 'Product id %d did not pass pre-build model filter for store %s', + $product->getId(), + $store->getCode() + ) + ); + } + try { + $nostoProduct->setUrl($this->urlBuilder->getUrlInStore($product, $store)); + $nostoProduct->setProductId((string)$product->getId()); + } catch (Exception $e) { + $message = sprintf("Could not build light product with id: %s", $product->getId()); + throw new NonBuildableProductException($message, $e); + } + $this->eventManager->dispatch( + 'nosto_product_load_after', + ['product' => $nostoProduct, 'magentoProduct' => $product, 'modelFilter' => $modelFilter] + ); + + return $nostoProduct; + } + /** * Adds selected attributes to all tags also in the custom fields section * diff --git a/Model/Service/Product/DefaultProductService.php b/Model/Service/Product/DefaultProductService.php index d6861bc04..068907ab0 100644 --- a/Model/Service/Product/DefaultProductService.php +++ b/Model/Service/Product/DefaultProductService.php @@ -108,4 +108,33 @@ public function getProduct(ProductInterface $product, StoreInterface $store) return null; } } + + /** + * @param ProductInterface $product + * @param StoreInterface $store + * @return NostoProduct|null + * @throws \Magento\Framework\Exception\NoSuchEntityException + */ + public function getLightProduct(ProductInterface $product, StoreInterface $store) + { + /** @var Product $product */ + /** @var Store $store */ + try { + $nostoProduct = $this->nostoProductBuilder->buildLightProduct( + $this->nostoProductRepository->reloadProduct( + $product->getId(), + $store->getId() + ), + $store + ); + return $nostoProduct; +// $nostoLightProduct = new stdClass(); +// $nostoLightProduct->product_id = $nostoProduct->getProductId(); +// $nostoLightProduct->url = $nostoProduct->getUrl(); +// return $nostoLightProduct; + } catch (NonBuildableProductException $e) { + $this->logger->exception($e); + return null; + } + } } diff --git a/Model/Service/Sync/Recrawl/RecrawlService.php b/Model/Service/Sync/Recrawl/RecrawlService.php new file mode 100644 index 000000000..11829a1c1 --- /dev/null +++ b/Model/Service/Sync/Recrawl/RecrawlService.php @@ -0,0 +1,181 @@ + + * @copyright 2020 Nosto Solutions Ltd + * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause + * + */ + +namespace Nosto\Tagging\Model\Service\Sync\Recrawl; + +use Exception; +use Magento\Catalog\Model\Product; +use Magento\Store\Model\Store; +use Nosto\Exception\MemoryOutOfBoundsException; +use Nosto\Model\Product\SkuCollection; +use Nosto\Model\Product\VariationCollection; +use Nosto\NostoException; +use Nosto\Operation\RecrawlProduct; +use Nosto\Tagging\Helper\Data as NostoDataHelper; +use Nosto\Tagging\Model\Product\Repository as ProductRepository; +use Nosto\Request\Http\Exception\AbstractHttpException; +use Nosto\Tagging\Helper\Account as NostoHelperAccount; +use Nosto\Tagging\Helper\Data as NostoHelperData; +use Nosto\Tagging\Helper\Url as NostoHelperUrl; +use Nosto\Tagging\Logger\Logger as NostoLogger; +use Nosto\Tagging\Model\ResourceModel\Magento\Product\Collection as ProductCollection; +use Nosto\Tagging\Model\Service\AbstractService; +use Nosto\Tagging\Model\Service\Cache\CacheService; +use Nosto\Tagging\Model\Service\Product\DefaultProductService; +use Nosto\Tagging\Util\PagingIterator; + +class RecrawlService extends AbstractService +{ + + public const BENCHMARK_RECRAWL_NAME = 'nosto_product_recrawl'; + public const BENCHMARK_RECRAWL_BREAKPOINT = 1; + public const PRODUCT_RECRAWL_BATCH_SIZE = 100; + + /** @var CacheService */ + private CacheService $cacheService; + + /** @var DefaultProductService */ + private DefaultProductService $productService; + + /** @var NostoHelperAccount */ + private NostoHelperAccount $nostoHelperAccount; + + /** @var NostoHelperUrl */ + private NostoHelperUrl $nostoHelperUrl; + + /** @var ProductRepository */ + private ProductRepository $productRepository; + + /** @var int */ + private int $recrawlBatchSize; + + /** + * RecrawlService constructor. + * @param CacheService $cacheService + * @param DefaultProductService $productService + * @param NostoHelperAccount $nostoHelperAccount + * @param NostoHelperData $nostoHelperData + * @param NostoHelperUrl $nostoHelperUrl + * @param ProductRepository $productRepository + * @param NostoLogger $logger + * @param int $recrawlBatchSize + */ + public function __construct( + CacheService $cacheService, + DefaultProductService $productService, + NostoHelperAccount $nostoHelperAccount, + NostoHelperData $nostoHelperData, + NostoHelperUrl $nostoHelperUrl, + ProductRepository $productRepository, + NostoLogger $logger, + int $recrawlBatchSize = self::PRODUCT_RECRAWL_BATCH_SIZE + ) { + $this->cacheService = $cacheService; + $this->productService = $productService; + $this->nostoHelperAccount = $nostoHelperAccount; + $this->nostoHelperUrl = $nostoHelperUrl; + $this->productRepository = $productRepository; + $this->recrawlBatchSize = $recrawlBatchSize; + parent::__construct($nostoHelperData, $nostoHelperAccount, $logger); + } + + /** + * Sends products ID's to Nosto to use the crawler to fetch product data + * @param ProductCollection $collection + * @param Store $store + * @throws MemoryOutOfBoundsException + * @throws NostoException + * @throws AbstractHttpException + * @throws Exception + */ + public function recrawl(ProductCollection $collection, Store $store) + { +// if (!$this->nostoDataHelper->isProductUpdatesEnabled($store)) { +// $this->logDebugWithStore( +// 'Nosto product sync is disabled - skipping products recrawl request to Nosto', +// $store +// ); +// return; +// } + $account = $this->nostoHelperAccount->findAccount($store); + $this->startBenchmark(self::BENCHMARK_RECRAWL_NAME, self::BENCHMARK_RECRAWL_BREAKPOINT); + + $index = 0; + $collection->setPageSize($this->recrawlBatchSize); + $iterator = new PagingIterator($collection); + + /** @var ProductCollection $page */ + foreach ($iterator as $page) { + $productIdsInBatch = []; + $this->checkMemoryConsumption('product recrawl request'); + $op = new RecrawlProduct($account, $this->nostoHelperUrl->getActiveDomain($store)); + $op->setResponseTimeout(60); + $products = $this->productRepository->getByIds( + $page->getAllIds( + $this->recrawlBatchSize, + ($iterator->getCurrentPageNumber() - 1) * $this->recrawlBatchSize + ) + ); + /** @var Product $product */ + foreach ($products->getItems() as $product) { + $productIdsInBatch[] = $product->getId(); + $nostoLightProduct = $this->productService->getLightProduct($product, $store); + if ($nostoLightProduct === null) { + throw new NostoException('Could not get product from the product service.'); + } + $nostoLightProduct->setSkus(new SkuCollection()); + $nostoLightProduct->setVariations(new VariationCollection()); + + $op->addProduct($nostoLightProduct); + // phpcs:ignore + $this->cacheService->save($nostoLightProduct, $store); + $this->tickBenchmark(self::BENCHMARK_RECRAWL_NAME); + } + + $this->logDebugWithStore( + sprintf( + 'Upserting batch of %d (%s) - API timeout is set to %d seconds', + $this->recrawlBatchSize, + implode(',', $productIdsInBatch), + 60 + ), + $store + ); + $op->requestRecrawl(); + } + $this->logBenchmarkSummary(self::BENCHMARK_RECRAWL_NAME, $store, $this); + } +} diff --git a/etc/di.xml b/etc/di.xml index cccf6dccd..19ac255f6 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -182,6 +182,9 @@ Nosto\Tagging\Model\Indexer\Dimensions\Product\ModeSwitcher + + Nosto\Tagging\Model\Indexer\Dimensions\Product\ModeSwitcher + @@ -199,6 +202,13 @@ + + + + Nosto\Tagging\Model\Indexer\Dimensions\StoreDimensionProvider + + + diff --git a/etc/indexer.xml b/etc/indexer.xml index 00183f6ee..b0a77060d 100644 --- a/etc/indexer.xml +++ b/etc/indexer.xml @@ -38,4 +38,9 @@ Nosto Product Indexer Populates message queue with product ids to be sent to Nosto + + + Nosto Product Indexer + Populates message queue with product ids to be sent to Nosto + diff --git a/etc/mview.xml b/etc/mview.xml index 232767d84..ab44e62bb 100644 --- a/etc/mview.xml +++ b/etc/mview.xml @@ -64,4 +64,34 @@ + + + +
+
+
+
+
+
+
+
+
+ + +
+ + +
+ + +
+
+ + +
+ + +
+ + From 8798f717f5d69a789631d678ba018c3d868870ba Mon Sep 17 00:00:00 2001 From: Cid Lopes Date: Mon, 30 Sep 2024 14:29:15 +0300 Subject: [PATCH 2/5] Update Model/Service/Product/DefaultProductService.php --- Model/Service/Product/DefaultProductService.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Model/Service/Product/DefaultProductService.php b/Model/Service/Product/DefaultProductService.php index 068907ab0..168597359 100644 --- a/Model/Service/Product/DefaultProductService.php +++ b/Model/Service/Product/DefaultProductService.php @@ -128,10 +128,6 @@ public function getLightProduct(ProductInterface $product, StoreInterface $store $store ); return $nostoProduct; -// $nostoLightProduct = new stdClass(); -// $nostoLightProduct->product_id = $nostoProduct->getProductId(); -// $nostoLightProduct->url = $nostoProduct->getUrl(); -// return $nostoLightProduct; } catch (NonBuildableProductException $e) { $this->logger->exception($e); return null; From 0d90835e94b7e09ef5500176c5a2ece8f4c8511c Mon Sep 17 00:00:00 2001 From: makso8makso Date: Mon, 7 Oct 2024 16:35:52 +0200 Subject: [PATCH 3/5] Phan Analysis - minor changes --- Model/Service/Product/DefaultProductService.php | 4 ++-- Model/Service/Sync/Recrawl/RecrawlService.php | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Model/Service/Product/DefaultProductService.php b/Model/Service/Product/DefaultProductService.php index 168597359..01a36faad 100644 --- a/Model/Service/Product/DefaultProductService.php +++ b/Model/Service/Product/DefaultProductService.php @@ -111,11 +111,11 @@ public function getProduct(ProductInterface $product, StoreInterface $store) /** * @param ProductInterface $product - * @param StoreInterface $store + * @param Store $store * @return NostoProduct|null * @throws \Magento\Framework\Exception\NoSuchEntityException */ - public function getLightProduct(ProductInterface $product, StoreInterface $store) + public function getLightProduct(ProductInterface $product, Store $store) { /** @var Product $product */ /** @var Store $store */ diff --git a/Model/Service/Sync/Recrawl/RecrawlService.php b/Model/Service/Sync/Recrawl/RecrawlService.php index 11829a1c1..4eb8bd4f9 100644 --- a/Model/Service/Sync/Recrawl/RecrawlService.php +++ b/Model/Service/Sync/Recrawl/RecrawlService.php @@ -44,7 +44,6 @@ use Nosto\Model\Product\VariationCollection; use Nosto\NostoException; use Nosto\Operation\RecrawlProduct; -use Nosto\Tagging\Helper\Data as NostoDataHelper; use Nosto\Tagging\Model\Product\Repository as ProductRepository; use Nosto\Request\Http\Exception\AbstractHttpException; use Nosto\Tagging\Helper\Account as NostoHelperAccount; From c3c8bf9513a25c523ee80c6b8950444df9fca2f3 Mon Sep 17 00:00:00 2001 From: makso8makso Date: Tue, 22 Oct 2024 11:43:04 +0200 Subject: [PATCH 4/5] added adminhtml configuration dropdown, update RecrawlService with dynamic values --- Helper/Data.php | 32 ++++++++++ Model/Config/Source/ProductsPerRequest.php | 59 +++++++++++++++++++ .../Source/TimeBetweenBatchOfRequests.php | 59 +++++++++++++++++++ Model/Service/Sync/Recrawl/RecrawlService.php | 21 +++---- etc/adminhtml/system.xml | 16 +++++ etc/config.xml | 2 + etc/indexer.xml | 2 +- 7 files changed, 178 insertions(+), 13 deletions(-) create mode 100644 Model/Config/Source/ProductsPerRequest.php create mode 100644 Model/Config/Source/TimeBetweenBatchOfRequests.php diff --git a/Helper/Data.php b/Helper/Data.php index 4e62e7706..b84ec85ae 100644 --- a/Helper/Data.php +++ b/Helper/Data.php @@ -128,6 +128,16 @@ class Data extends AbstractHelper */ public const XML_PATH_INDEXER_MEMORY = 'nosto/flags/indexer_memory'; + /** + * Product per request + */ + public const XML_PATH_PRODUCT_PER_REQUEST = 'nosto/flags/product_per_request'; + + /** + * Request timeout + */ + public const XML_PATH_REQUEST_TIMEOUT = 'nosto/flags/request_timeout'; + /** * Path to the configuration object that stores the preference for indexing disabled products */ @@ -441,6 +451,28 @@ public function getIndexerMemory(StoreInterface $store = null) return $this->getStoreConfig(self::XML_PATH_INDEXER_MEMORY, $store); } + /** + * Products per request + * + * @param StoreInterface|null $store the store model or null. + * @return int the configuration value + */ + public function getProductsPerRequest(StoreInterface $store = null) + { + return $this->getStoreConfig(self::XML_PATH_PRODUCT_PER_REQUEST, $store); + } + + /** + * Request timeout + * + * @param StoreInterface|null $store the store model or null. + * @return int the configuration value + */ + public function getRequestTimeout(StoreInterface $store = null) + { + return $this->getStoreConfig(self::XML_PATH_REQUEST_TIMEOUT, $store); + } + /** * Returns maximum percentage of PHP available memory that indexer should use * diff --git a/Model/Config/Source/ProductsPerRequest.php b/Model/Config/Source/ProductsPerRequest.php new file mode 100644 index 000000000..9225e6613 --- /dev/null +++ b/Model/Config/Source/ProductsPerRequest.php @@ -0,0 +1,59 @@ + + * @copyright 2020 Nosto Solutions Ltd + * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause + * + */ + +namespace Nosto\Tagging\Model\Config\Source; + +use Magento\Framework\Data\OptionSourceInterface; +use Magento\Framework\Phrase; + +class ProductsPerRequest implements OptionSourceInterface +{ + /** + * Options getter + * + * @return array + */ + public function toOptionArray() + { + return [ + ['value' => 100, 'label' => new Phrase('100')], + ['value' => 200, 'label' => new Phrase('200')], + ['value' => 300, 'label' => new Phrase('300')], + ['value' => 400, 'label' => new Phrase('400')], + ['value' => 500, 'label' => new Phrase('500')], + ]; + } +} diff --git a/Model/Config/Source/TimeBetweenBatchOfRequests.php b/Model/Config/Source/TimeBetweenBatchOfRequests.php new file mode 100644 index 000000000..95db5a8da --- /dev/null +++ b/Model/Config/Source/TimeBetweenBatchOfRequests.php @@ -0,0 +1,59 @@ + + * @copyright 2020 Nosto Solutions Ltd + * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause + * + */ + +namespace Nosto\Tagging\Model\Config\Source; + +use Magento\Framework\Data\OptionSourceInterface; +use Magento\Framework\Phrase; + +class TimeBetweenBatchOfRequests implements OptionSourceInterface +{ + /** + * Options getter + * + * @return array + */ + public function toOptionArray() + { + return [ + ['value' => 60, 'label' => new Phrase('60')], + ['value' => 120, 'label' => new Phrase('120')], + ['value' => 180, 'label' => new Phrase('180')], + ['value' => 240, 'label' => new Phrase('240')], + ['value' => 300, 'label' => new Phrase('300')], + ]; + } +} diff --git a/Model/Service/Sync/Recrawl/RecrawlService.php b/Model/Service/Sync/Recrawl/RecrawlService.php index 4eb8bd4f9..b30870fd6 100644 --- a/Model/Service/Sync/Recrawl/RecrawlService.php +++ b/Model/Service/Sync/Recrawl/RecrawlService.php @@ -61,7 +61,6 @@ class RecrawlService extends AbstractService public const BENCHMARK_RECRAWL_NAME = 'nosto_product_recrawl'; public const BENCHMARK_RECRAWL_BREAKPOINT = 1; - public const PRODUCT_RECRAWL_BATCH_SIZE = 100; /** @var CacheService */ private CacheService $cacheService; @@ -78,9 +77,6 @@ class RecrawlService extends AbstractService /** @var ProductRepository */ private ProductRepository $productRepository; - /** @var int */ - private int $recrawlBatchSize; - /** * RecrawlService constructor. * @param CacheService $cacheService @@ -90,7 +86,6 @@ class RecrawlService extends AbstractService * @param NostoHelperUrl $nostoHelperUrl * @param ProductRepository $productRepository * @param NostoLogger $logger - * @param int $recrawlBatchSize */ public function __construct( CacheService $cacheService, @@ -99,15 +94,13 @@ public function __construct( NostoHelperData $nostoHelperData, NostoHelperUrl $nostoHelperUrl, ProductRepository $productRepository, - NostoLogger $logger, - int $recrawlBatchSize = self::PRODUCT_RECRAWL_BATCH_SIZE + NostoLogger $logger ) { $this->cacheService = $cacheService; $this->productService = $productService; $this->nostoHelperAccount = $nostoHelperAccount; $this->nostoHelperUrl = $nostoHelperUrl; $this->productRepository = $productRepository; - $this->recrawlBatchSize = $recrawlBatchSize; parent::__construct($nostoHelperData, $nostoHelperAccount, $logger); } @@ -132,8 +125,11 @@ public function recrawl(ProductCollection $collection, Store $store) $account = $this->nostoHelperAccount->findAccount($store); $this->startBenchmark(self::BENCHMARK_RECRAWL_NAME, self::BENCHMARK_RECRAWL_BREAKPOINT); + $timeBetweenBatchOfRequests = $this->getDataHelper()->getRequestTimeout(); + $productsPerRequest = $this->getDataHelper()->getProductsPerRequest(); + $index = 0; - $collection->setPageSize($this->recrawlBatchSize); + $collection->setPageSize($productsPerRequest); $iterator = new PagingIterator($collection); /** @var ProductCollection $page */ @@ -144,8 +140,8 @@ public function recrawl(ProductCollection $collection, Store $store) $op->setResponseTimeout(60); $products = $this->productRepository->getByIds( $page->getAllIds( - $this->recrawlBatchSize, - ($iterator->getCurrentPageNumber() - 1) * $this->recrawlBatchSize + $productsPerRequest, + ($iterator->getCurrentPageNumber() - 1) * $productsPerRequest ) ); /** @var Product $product */ @@ -167,13 +163,14 @@ public function recrawl(ProductCollection $collection, Store $store) $this->logDebugWithStore( sprintf( 'Upserting batch of %d (%s) - API timeout is set to %d seconds', - $this->recrawlBatchSize, + $productsPerRequest, implode(',', $productIdsInBatch), 60 ), $store ); $op->requestRecrawl(); + sleep($timeBetweenBatchOfRequests); } $this->logBenchmarkSummary(self::BENCHMARK_RECRAWL_NAME, $store, $this); } diff --git a/etc/adminhtml/system.xml b/etc/adminhtml/system.xml index 44e6efee5..5515e3001 100644 --- a/etc/adminhtml/system.xml +++ b/etc/adminhtml/system.xml @@ -191,6 +191,22 @@ Nosto\Tagging\Model\Config\Source\Memory + + + + + + Nosto\Tagging\Model\Config\Source\ProductsPerRequest + + + + + + + Nosto\Tagging\Model\Config\Source\TimeBetweenBatchOfRequests + diff --git a/etc/config.xml b/etc/config.xml index 8848eaa1d..8e735381a 100644 --- a/etc/config.xml +++ b/etc/config.xml @@ -56,6 +56,8 @@ 0 1 50 + 100 + 60 0 0 1 diff --git a/etc/indexer.xml b/etc/indexer.xml index b0a77060d..4a068dc37 100644 --- a/etc/indexer.xml +++ b/etc/indexer.xml @@ -40,7 +40,7 @@ - Nosto Product Indexer + Nosto Product Light Indexer Populates message queue with product ids to be sent to Nosto From 4c0dde8ed6d825681b882e3914715b9d0ea586f3 Mon Sep 17 00:00:00 2001 From: makso8makso Date: Wed, 27 Nov 2024 10:39:49 +0100 Subject: [PATCH 5/5] create a mechanism, added a warning on adminhtml panel --- Helper/Data.php | 54 +++++++++- Model/Config/Backend/MultiIndexer.php | 100 ++++++++++++++++++ Model/Config/Source/MultiIndexer.php | 62 +++++++++++ Model/Config/Source/WebsitePublicity.php | 33 ++++++ Model/Indexer/LightProductIndexer.php | 13 ++- Model/Indexer/ProductIndexer.php | 22 ++-- Model/Service/AbstractService.php | 2 +- Model/Service/Sync/Recrawl/RecrawlService.php | 18 ++-- Model/Service/Sync/Upsert/SyncService.php | 2 +- composer.json | 2 +- etc/adminhtml/system.xml | 83 ++++++++++----- etc/config.xml | 9 +- 12 files changed, 348 insertions(+), 52 deletions(-) create mode 100644 Model/Config/Backend/MultiIndexer.php create mode 100644 Model/Config/Source/MultiIndexer.php create mode 100644 Model/Config/Source/WebsitePublicity.php diff --git a/Helper/Data.php b/Helper/Data.php index b84ec85ae..eceff1ff3 100644 --- a/Helper/Data.php +++ b/Helper/Data.php @@ -179,9 +179,9 @@ class Data extends AbstractHelper public const XML_PATH_MULTI_CURRENCY = 'nosto/multicurrency/method'; /** - * Path to the configuration object for light indexer + * Path to the configuration object for multi currency */ - public const XML_PATH_USE_LIGHT_INDEXER = 'nosto/flags/use_light_indexer'; + public const XML_PATH_MULTI_INDEXER = 'nosto/product_sync/indexer_method'; /** * @var string Nosto customer reference attribute name @@ -203,6 +203,14 @@ class Data extends AbstractHelper public const SETTING_VALUE_MC_DISABLED = 'disabled'; public const SETTING_VALUE_MC_UNDEFINED = 'undefined'; + /** + * Values of the multi indexer settings + */ + public const SETTING_VALUE_MI_PRODUCT_INDEXER = 'productindexer'; + public const SETTING_VALUE_MI_PRODUCT_LIGHT_INDEXER = 'productlightindexer'; + public const SETTING_VALUE_MI_PRODUCT_DISABLED = 'disabled'; + public const SETTING_VALUE_MI_UNDEFINDED = 'undefined'; + /** * Name of the module */ @@ -542,14 +550,50 @@ public function getMultiCurrencyMethod(StoreInterface $store = null) } /** - * Returns if the light indexer should be used + * Returns if multi indexer is disabled + * + * @param StoreInterface|null $store the store model or null. + * @return bool the configuration value + */ + public function isMultiIndexerDisabled(StoreInterface $store = null) + { + $storeConfig = $this->getMultiIndexerMethod($store); + return ($storeConfig === self::SETTING_VALUE_MI_PRODUCT_DISABLED); + } + + /** + * Returns if multi indexer is enabled + * + * @param StoreInterface|null $store the store model or null. + * @return bool the configuration value + */ + public function isMultiIndexerMainIndexerEnabled(StoreInterface $store = null) + { + $storeConfig = $this->getMultiIndexerMethod($store); + return ($storeConfig === self::SETTING_VALUE_MI_PRODUCT_INDEXER); + } + + /** + * Returns if multi indexer is enabled + * + * @param StoreInterface|null $store the store model or null. + * @return bool the configuration value + */ + public function isMultiIndexerLightIndexerEnabled(StoreInterface $store = null) + { + $storeConfig = $this->getMultiIndexerMethod($store); + return ($storeConfig === self::SETTING_VALUE_MI_PRODUCT_LIGHT_INDEXER); + } + + /** + * Returns the multi indexer setup value / multi indexer_method * * @param StoreInterface|null $store the store model or null. * @return string the configuration value */ - public function getUseLightIndexer(StoreInterface $store = null) + public function getMultiIndexerMethod(StoreInterface $store = null) { - return $this->getStoreConfig(self::XML_PATH_USE_LIGHT_INDEXER, $store); + return $this->getStoreConfig(self::XML_PATH_MULTI_INDEXER, $store); } /** diff --git a/Model/Config/Backend/MultiIndexer.php b/Model/Config/Backend/MultiIndexer.php new file mode 100644 index 000000000..69646a167 --- /dev/null +++ b/Model/Config/Backend/MultiIndexer.php @@ -0,0 +1,100 @@ + + * @copyright 2020 Nosto Solutions Ltd + * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause + * + */ + +namespace Nosto\Tagging\Model\Config\Backend; + +use Magento\Framework\App\Cache\TypeListInterface; +use Magento\Framework\App\Config\ScopeConfigInterface; +use Magento\Framework\App\Config\Storage\WriterInterface; +use Magento\Framework\App\Config\Value; +use Magento\Framework\Data\Collection\AbstractDb; +use Magento\Framework\Model\Context; +use Magento\Framework\Model\ResourceModel\AbstractResource; +use Magento\Framework\Registry; +use Nosto\Tagging\Helper\Data as NostoHelperData; + +class MultiIndexer extends Value +{ + private WriterInterface $configWriter; + + /** + * MultiCurrency constructor. + * @param Context $context + * @param Registry $registry + * @param ScopeConfigInterface $config + * @param TypeListInterface $cacheTypeList + * @param WriterInterface $configWriter + * @param AbstractResource|null $resource + * @param AbstractDb|null $resourceCollection + * @param array $data + */ + public function __construct( + Context $context, + Registry $registry, + ScopeConfigInterface $config, + TypeListInterface $cacheTypeList, + WriterInterface $configWriter, + AbstractResource $resource = null, + AbstractDb $resourceCollection = null, + array $data = [] + ) { + $this->configWriter = $configWriter; + parent::__construct($context, $registry, $config, $cacheTypeList, $resource, $resourceCollection, $data); + } + + /** + * @return Value + */ + public function beforeSave() //@codingStandardsIgnoreLine + { + $value = $this->getValue(); + $scopeType = $this->getScope(); + $scopeId = $this->getScopeId(); + + if ($value == NostoHelperData::SETTING_VALUE_MI_PRODUCT_INDEXER + || $value == NostoHelperData::SETTING_VALUE_MI_PRODUCT_LIGHT_INDEXER + ) { +// $this->configWriter->save( +// NostoHelperData::XML_PATH_PRICING_VARIATION, +// '0', +// $scopeType, +// $scopeId +// ); + } + + return parent::beforeSave(); + } +} diff --git a/Model/Config/Source/MultiIndexer.php b/Model/Config/Source/MultiIndexer.php new file mode 100644 index 000000000..0ca15f0b2 --- /dev/null +++ b/Model/Config/Source/MultiIndexer.php @@ -0,0 +1,62 @@ + + * @copyright 2020 Nosto Solutions Ltd + * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause + * + */ + +namespace Nosto\Tagging\Model\Config\Source; + +use Magento\Framework\Data\OptionSourceInterface; +use Magento\Framework\Phrase; +use Nosto\Tagging\Helper\Data; + +/** + * Option array class to generate a list of selectable options that allows the merchant to choose + * any image attribute for his image tag. + */ +class MultiIndexer implements OptionSourceInterface +{ + /** + * Options getter + * + * @return array + */ + public function toOptionArray() + { + return [ + ['value' => Data::SETTING_VALUE_MI_PRODUCT_INDEXER, 'label' => new Phrase('Product Indexer')], + ['value' => Data::SETTING_VALUE_MI_PRODUCT_LIGHT_INDEXER, 'label' => new Phrase('Product light indexer')], + ['value' => Data::SETTING_VALUE_MI_PRODUCT_DISABLED, 'label' => new Phrase('Disabled')] + ]; + } +} diff --git a/Model/Config/Source/WebsitePublicity.php b/Model/Config/Source/WebsitePublicity.php new file mode 100644 index 000000000..c897c3228 --- /dev/null +++ b/Model/Config/Source/WebsitePublicity.php @@ -0,0 +1,33 @@ + 1, 'label' => __('Yes')], ['value' => 0, 'label' => __('No')]]; + } + + /** + * Get options in "key-value" format + * + * @return array + */ + public function toArray() + { + return [0 => __('No'), 1 => __('Yes')]; + } +} diff --git a/Model/Indexer/LightProductIndexer.php b/Model/Indexer/LightProductIndexer.php index c92ef4238..c96fe21ab 100644 --- a/Model/Indexer/LightProductIndexer.php +++ b/Model/Indexer/LightProductIndexer.php @@ -51,6 +51,7 @@ use Nosto\Tagging\Model\Service\Indexer\IndexerStatusServiceInterface; use Symfony\Component\Console\Input\InputInterface; use Nosto\Tagging\Model\Service\Sync\Recrawl\RecrawlService; +use Nosto\Tagging\Helper\Data as NostoHelperData; /** * Class ProductIndexer @@ -66,6 +67,9 @@ class LightProductIndexer extends AbstractIndexer /** @var ProductModeSwitcher */ private ProductModeSwitcher $modeSwitcher; + /** @var NostoHelperData */ + protected NostoHelperData $nostoHelperData; + /** @var RecrawlService */ private RecrawlService $recrawlService; @@ -81,6 +85,7 @@ class LightProductIndexer extends AbstractIndexer * @param InputInterface $input * @param IndexerStatusServiceInterface $indexerStatusService * @param RecrawlService $recrawlService + * @param NostoHelperData $nostoHelperData */ public function __construct( NostoHelperScope $nostoHelperScope, @@ -92,11 +97,13 @@ public function __construct( ProcessManager $processManager, InputInterface $input, IndexerStatusServiceInterface $indexerStatusService, - RecrawlService $recrawlService + RecrawlService $recrawlService, + NostoHelperData $nostoHelperData ) { $this->productCollectionBuilder = $productCollectionBuilder; $this->modeSwitcher = $modeSwitcher; $this->recrawlService = $recrawlService; + $this->nostoHelperData = $nostoHelperData; parent::__construct( $nostoHelperScope, $logger, @@ -123,10 +130,10 @@ public function getModeSwitcher(): ModeSwitcherInterface */ public function doIndex(Store $store, array $ids = []) { -// if ($this->nostoHelperData->getUseLightIndexer($store)) { + if ($this->nostoHelperData->isMultiIndexerLightIndexerEnabled($store)) { $collection = $this->getCollection($store, $ids); $this->recrawlService->recrawl($collection, $store); -// } + } } /** diff --git a/Model/Indexer/ProductIndexer.php b/Model/Indexer/ProductIndexer.php index c6dc1e725..71b6bd549 100644 --- a/Model/Indexer/ProductIndexer.php +++ b/Model/Indexer/ProductIndexer.php @@ -41,6 +41,7 @@ use Magento\Store\Model\App\Emulation; use Magento\Store\Model\Store; use Nosto\NostoException; +use Nosto\Tagging\Helper\Data as NostoHelperData; use Nosto\Tagging\Helper\Scope as NostoHelperScope; use Nosto\Tagging\Logger\Logger as NostoLogger; use Nosto\Tagging\Model\Indexer\Dimensions\Product\ModeSwitcher as ProductModeSwitcher; @@ -69,6 +70,9 @@ class ProductIndexer extends AbstractIndexer /** @var ProductModeSwitcher */ private ProductModeSwitcher $modeSwitcher; + /** @var NostoHelperData */ + protected NostoHelperData $nostoHelperData; + /** * Invalidate constructor. * @param NostoHelperScope $nostoHelperScope @@ -92,11 +96,13 @@ public function __construct( Emulation $storeEmulation, ProcessManager $processManager, InputInterface $input, - IndexerStatusServiceInterface $indexerStatusService + IndexerStatusServiceInterface $indexerStatusService, + NostoHelperData $nostoHelperData ) { $this->productUpdateService = $productUpdateService; $this->productCollectionBuilder = $productCollectionBuilder; $this->modeSwitcher = $modeSwitcher; + $this->nostoHelperData = $nostoHelperData; parent::__construct( $nostoHelperScope, $logger, @@ -123,11 +129,15 @@ public function getModeSwitcher(): ModeSwitcherInterface */ public function doIndex(Store $store, array $ids = []) { - $collection = $this->getCollection($store, $ids); - $this->productUpdateService->addCollectionToUpdateMessageQueue( - $collection, - $store - ); + if ($this->nostoHelperData->isMultiIndexerMainIndexerEnabled($store)) { + $collection = $this->getCollection($store, $ids); + $this->productUpdateService->addCollectionToUpdateMessageQueue( + $collection, + $store + ); + } else { + $this->nostoLogger->debugWithSource(); + } } /** diff --git a/Model/Service/AbstractService.php b/Model/Service/AbstractService.php index 6351e5b3a..d019fb124 100644 --- a/Model/Service/AbstractService.php +++ b/Model/Service/AbstractService.php @@ -49,7 +49,7 @@ abstract class AbstractService { /** @var NostoDataHelper */ - private NostoDataHelper $nostoDataHelper; + protected NostoDataHelper $nostoDataHelper; /** @var NostoLogger */ private NostoLogger $nostoLogger; diff --git a/Model/Service/Sync/Recrawl/RecrawlService.php b/Model/Service/Sync/Recrawl/RecrawlService.php index b30870fd6..7d4434a97 100644 --- a/Model/Service/Sync/Recrawl/RecrawlService.php +++ b/Model/Service/Sync/Recrawl/RecrawlService.php @@ -71,6 +71,9 @@ class RecrawlService extends AbstractService /** @var NostoHelperAccount */ private NostoHelperAccount $nostoHelperAccount; + /** @var NostoHelperData */ + protected NostoHelperData $nostoHelperData; + /** @var NostoHelperUrl */ private NostoHelperUrl $nostoHelperUrl; @@ -99,6 +102,7 @@ public function __construct( $this->cacheService = $cacheService; $this->productService = $productService; $this->nostoHelperAccount = $nostoHelperAccount; + $this->nostoHelperData = $nostoHelperData; $this->nostoHelperUrl = $nostoHelperUrl; $this->productRepository = $productRepository; parent::__construct($nostoHelperData, $nostoHelperAccount, $logger); @@ -115,13 +119,13 @@ public function __construct( */ public function recrawl(ProductCollection $collection, Store $store) { -// if (!$this->nostoDataHelper->isProductUpdatesEnabled($store)) { -// $this->logDebugWithStore( -// 'Nosto product sync is disabled - skipping products recrawl request to Nosto', -// $store -// ); -// return; -// } + if (!$this->nostoDataHelper->isProductUpdatesEnabled($store)) { + $this->logDebugWithStore( + 'Nosto product sync is disabled - skipping products recrawl request to Nosto', + $store + ); + return; + } $account = $this->nostoHelperAccount->findAccount($store); $this->startBenchmark(self::BENCHMARK_RECRAWL_NAME, self::BENCHMARK_RECRAWL_BREAKPOINT); diff --git a/Model/Service/Sync/Upsert/SyncService.php b/Model/Service/Sync/Upsert/SyncService.php index fda89b5ae..de40ae3e4 100644 --- a/Model/Service/Sync/Upsert/SyncService.php +++ b/Model/Service/Sync/Upsert/SyncService.php @@ -66,7 +66,7 @@ class SyncService extends AbstractService private NostoHelperUrl $nostoHelperUrl; /** @var NostoDataHelper */ - private NostoDataHelper $nostoDataHelper; + protected NostoDataHelper $nostoDataHelper; /** @var ProductServiceInterface */ private ProductServiceInterface $productService; diff --git a/composer.json b/composer.json index 86278a314..e0f1ab983 100644 --- a/composer.json +++ b/composer.json @@ -41,7 +41,7 @@ "php": ">=7.4.0", "magento/framework": ">=101.0.6|~104.0", "ext-json": "*", - "nosto/php-sdk": "^7.4", + "nosto/php-sdk": "dev-add-recrawl-endpoint", "laminas/laminas-uri": "*" }, "repositories": [ diff --git a/etc/adminhtml/system.xml b/etc/adminhtml/system.xml index 5515e3001..02c829f33 100644 --- a/etc/adminhtml/system.xml +++ b/etc/adminhtml/system.xml @@ -159,16 +159,6 @@ Magento\Config\Model\Config\Source\Yesno - - - Warning! Disabling this feature may - lead to products with incorrect pricing and availability data to be displayed - in recommendations.]]> - - Magento\Config\Model\Config\Source\Yesno - @@ -191,22 +181,6 @@ Nosto\Tagging\Model\Config\Source\Memory - - - - - - Nosto\Tagging\Model\Config\Source\ProductsPerRequest - - - - - - - Nosto\Tagging\Model\Config\Source\TimeBetweenBatchOfRequests - @@ -300,6 +274,63 @@ Nosto\Tagging\Block\Adminhtml\Form\Field\Tokens + + + + + Set this to "Product indexer" if your store uses Magento's product regural indexer. If the store view uses light indexer set this to + "Product light indexer". If you disable indexer you set this to "Disabled" and Nosto will not try to do any indexer. + + Nosto\Tagging\Model\Config\Source\MultiIndexer + Nosto\Tagging\Model\Config\Backend\MultiIndexer + + + + productindexer + + + Warning! Disabling this feature may + lead to products with incorrect pricing and availability data to be displayed + in recommendations.]]> + + Magento\Config\Model\Config\Source\Yesno + + + + productlightindexer + + + + Warning! The light recrawl indexer does not currently perform partial API updates with inventory levels. This feature is not yet implemented on Nosto's side.]]> + + Nosto\Tagging\Model\Config\Source\ProductsPerRequest + + + + productlightindexer + + + + Warning! The light recrawl indexer does not currently perform partial API updates with inventory levels. This feature is not yet implemented on Nosto's side.]]> + + Nosto\Tagging\Model\Config\Source\TimeBetweenBatchOfRequests + + + + Set this yes, because the light recrawl indexer only works if the website is publicity + + Nosto\Tagging\Model\Config\Source\WebsitePublicity + + diff --git a/etc/config.xml b/etc/config.xml index 8e735381a..7dfb2fcbc 100644 --- a/etc/config.xml +++ b/etc/config.xml @@ -46,6 +46,13 @@ 1 + + undefined + 1 + 100 + 60 + 1 + 1 1 @@ -56,8 +63,6 @@ 0 1 50 - 100 - 60 0 0 1