diff --git a/Model/Indexer/ProductIndexer.php b/Model/Indexer/ProductIndexer.php index c6dc1e725..fedc18803 100644 --- a/Model/Indexer/ProductIndexer.php +++ b/Model/Indexer/ProductIndexer.php @@ -128,6 +128,38 @@ public function doIndex(Store $store, array $ids = []) $collection, $store ); + $this->handleDeletedProducts($collection, $store, $ids); + } + + /** + * @param ProductCollection $existingCollection + * @param Store $store + * @param array $givenIds + * @throws NostoException + */ + private function handleDeletedProducts(ProductCollection $existingCollection, Store $store, array $givenIds) + { + if (!empty($givenIds)) { + $existingCollection->setPageSize(1000); + $iterator = new PagingIterator($existingCollection); + $present = []; + foreach ($iterator as $page) { + foreach ($page->getItems() as $item) { + /** @noinspection PhpPossiblePolymorphicInvocationInspection */ + $id = $item->getId(); + $present[$id] = $id; + } + } + $removed = []; + foreach ($givenIds as $productId) { + if (!isset($present[$productId])) { + $removed[] = $productId; + } + } + if (count($removed) > 0) { + $this->productUpdateService->addIdsToDeleteMessageQueue($removed, $store); + } + } } /** diff --git a/Model/Service/Sync/Delete/AsyncBulkConsumer.php b/Model/Service/Sync/Delete/AsyncBulkConsumer.php new file mode 100644 index 000000000..0b067ebb5 --- /dev/null +++ b/Model/Service/Sync/Delete/AsyncBulkConsumer.php @@ -0,0 +1,93 @@ + + * @copyright 2020 Nosto Solutions Ltd + * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause + * + */ + +namespace Nosto\Tagging\Model\Service\Sync\Delete; + +use Nosto\NostoException; +use Nosto\Tagging\Model\Service\Sync\AbstractBulkConsumer; +use Nosto\Tagging\Helper\Scope as NostoHelperScope; +use Magento\Framework\EntityManager\EntityManager; +use Magento\Framework\Json\Helper\Data as JsonHelper; +use Nosto\Tagging\Logger\Logger; +use Magento\Store\Model\App\Emulation; + +class AsyncBulkConsumer extends AbstractBulkConsumer +{ + /** @var DeleteService */ + private DeleteService $deleteService; + + /** @var NostoHelperScope */ + private NostoHelperScope $nostoHelperScope; + + /** + * AsyncBulkConsumer constructor. + * @param DeleteService $deleteService + * @param NostoHelperScope $nostoHelperScope + * @param JsonHelper $jsonHelper + * @param EntityManager $entityManager + * @param Emulation $storeEmulation + * @param Logger $logger + */ + public function __construct( + DeleteService $deleteService, + NostoHelperScope $nostoHelperScope, + JsonHelper $jsonHelper, + EntityManager $entityManager, + Emulation $storeEmulation, + Logger $logger + ) { + $this->deleteService = $deleteService; + $this->nostoHelperScope = $nostoHelperScope; + parent::__construct( + $logger, + $jsonHelper, + $entityManager, + $storeEmulation + ); + } + + /** + * @inheritDoc + * @param array $productIds + * @param string $storeId + * @throws NostoException + */ + public function doOperation(array $productIds, string $storeId) + { + $store = $this->nostoHelperScope->getStore($storeId); + $this->deleteService->delete($productIds, $store); + } +} diff --git a/Model/Service/Sync/Delete/AsyncBulkPublisher.php b/Model/Service/Sync/Delete/AsyncBulkPublisher.php new file mode 100644 index 000000000..5198406c7 --- /dev/null +++ b/Model/Service/Sync/Delete/AsyncBulkPublisher.php @@ -0,0 +1,77 @@ + + * @copyright 2020 Nosto Solutions Ltd + * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause + * + */ + +namespace Nosto\Tagging\Model\Service\Sync\Delete; + +use Nosto\Tagging\Model\Service\Sync\AbstractBulkPublisher; + +class AsyncBulkPublisher extends AbstractBulkPublisher +{ + public const NOSTO_DELETE_MESSAGE_QUEUE = 'nosto_product_sync.delete'; + public const BULK_SIZE = 100; + + /** + * @inheritDoc + */ + public function getTopicName(): string + { + return self::NOSTO_DELETE_MESSAGE_QUEUE; + } + + /** + * @inheritDoc + */ + public function getBulkSize(): int + { + return self::BULK_SIZE; + } + + /** + * @inheritDoc + */ + public function getBulkDescription(): string + { + return sprintf('Delete %d Nosto products', 2); + } + + /** + * @inheritDoc + */ + public function getMetaData(): string + { + return 'Delete Nosto products'; + } +} diff --git a/Model/Service/Sync/Delete/DeleteService.php b/Model/Service/Sync/Delete/DeleteService.php new file mode 100644 index 000000000..aafb1f39a --- /dev/null +++ b/Model/Service/Sync/Delete/DeleteService.php @@ -0,0 +1,134 @@ + + * @copyright 2020 Nosto Solutions Ltd + * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause + * + */ + +namespace Nosto\Tagging\Model\Service\Sync\Delete; + +use Exception; +use Magento\Store\Model\Store; +use Nosto\Model\Signup\Account as NostoSignupAccount; +use Nosto\NostoException; +use Nosto\Operation\DeleteProduct; +use Nosto\Tagging\Helper\Account as NostoHelperAccount; +use Nosto\Tagging\Helper\Data as NostoHelperData; +use Nosto\Tagging\Helper\Url as NostoHelperUrl; +use Nosto\Tagging\Logger\Logger as NostoLogger; +use Nosto\Tagging\Model\Service\AbstractService; +use Nosto\Tagging\Model\Service\Cache\CacheService; + +class DeleteService extends AbstractService +{ + + public const BENCHMARK_DELETE_NAME = 'nosto_product_delete'; + public const BENCHMARK_DELETE_BREAKPOINT = 1; + public const PRODUCT_DELETION_BATCH_SIZE = 100; + + /** @var CacheService */ + private CacheService $cacheService; + + /** @var NostoHelperAccount */ + private NostoHelperAccount $nostoHelperAccount; + + /** @var NostoHelperUrl */ + private NostoHelperUrl $nostoHelperUrl; + + /** @var int */ + private int $deleteBatchSize; + + /** + * DeleteService constructor. + * @param CacheService $cacheService + * @param NostoHelperAccount $nostoHelperAccount + * @param NostoHelperData $nostoHelperData + * @param NostoHelperUrl $nostoHelperUrl + * @param NostoLogger $logger + * @param $deleteBatchSize + */ + public function __construct( + CacheService $cacheService, + NostoHelperAccount $nostoHelperAccount, + NostoHelperData $nostoHelperData, + NostoHelperUrl $nostoHelperUrl, + NostoLogger $logger, + $deleteBatchSize + ) { + $this->cacheService = $cacheService; + $this->nostoHelperAccount = $nostoHelperAccount; + $this->nostoHelperUrl = $nostoHelperUrl; + $this->deleteBatchSize = $deleteBatchSize; + parent::__construct($nostoHelperData, $nostoHelperAccount, $logger); + } + + /** + * Discontinues products in Nosto and removes indexed products from Nosto product index + * + * @param array $productIds + * @param Store $store + * @throws NostoException + */ + public function delete(array $productIds, Store $store) + { + if (count($productIds) === 0) { + return; + } + $account = $this->nostoHelperAccount->findAccount($store); + if ($account instanceof NostoSignupAccount === false) { + throw new NostoException(sprintf('Store view %s does not have Nosto installed', $store->getName())); + } + $this->startBenchmark(self::BENCHMARK_DELETE_NAME, self::BENCHMARK_DELETE_BREAKPOINT); + $productIdBatches = array_chunk($productIds, $this->deleteBatchSize); + $this->logDebugWithStore( + sprintf( + 'Deleting total of %d products in batches of %d', + count($productIds), + count($productIdBatches) + ), + $store + ); + foreach ($productIdBatches as $ids) { + try { + $op = new DeleteProduct($account, $this->nostoHelperUrl->getActiveDomain($store)); + $op->setResponseTimeout(30); + $op->setProductIds($ids); + $op->delete(); // @codingStandardsIgnoreLine + $this->cacheService->removeByProductIds($store, $ids); + $this->tickBenchmark(self::BENCHMARK_DELETE_NAME); + } catch (Exception $e) { + $this->getLogger()->exception($e); + } + } + $this->logBenchmarkSummary(self::BENCHMARK_DELETE_NAME, $store); + } +} diff --git a/Model/Service/Update/ProductUpdateService.php b/Model/Service/Update/ProductUpdateService.php index 12c6a6946..2e5a4d8ed 100644 --- a/Model/Service/Update/ProductUpdateService.php +++ b/Model/Service/Update/ProductUpdateService.php @@ -61,6 +61,9 @@ class ProductUpdateService extends AbstractService /** @var BulkPublisherInterface */ private BulkPublisherInterface $upsertBulkPublisher; + /** @var BulkPublisherInterface */ + private BulkPublisherInterface $deleteBulkPublisher; + /** * ProductUpdateService constructor. * @param NostoLogger $logger @@ -68,6 +71,7 @@ class ProductUpdateService extends AbstractService * @param NostoAccountHelper $nostoAccountHelper * @param NostoProductRepository $nostoProductRepository * @param BulkPublisherInterface $upsertBulkPublisher + * @param BulkPublisherInterface $deleteBulkPublisher * @param int $batchSize */ public function __construct( @@ -76,11 +80,13 @@ public function __construct( NostoAccountHelper $nostoAccountHelper, NostoProductRepository $nostoProductRepository, BulkPublisherInterface $upsertBulkPublisher, + BulkPublisherInterface $deleteBulkPublisher, int $batchSize ) { parent::__construct($nostoDataHelper, $nostoAccountHelper, $logger); $this->nostoProductRepository = $nostoProductRepository; $this->upsertBulkPublisher = $upsertBulkPublisher; + $this->deleteBulkPublisher = $deleteBulkPublisher; $this->batchSize = $batchSize; } @@ -117,6 +123,24 @@ public function addCollectionToUpdateMessageQueue(ProductCollection $collection, } } + /** + * Sets the product ids into the delete message queue + * + * @param $productIds + * @param Store $store + */ + public function addIdsToDeleteMessageQueue($productIds, Store $store) + { + if ($this->getAccountHelper()->findAccount($store) === null) { + $this->logDebugWithStore('No nosto account found for the store', $store); + return; + } + $batchedIds = array_chunk($productIds, $this->batchSize); + foreach ($batchedIds as $idBatch) { + $this->deleteBulkPublisher->execute($store->getId(), $idBatch); + } + } + /** * @param ProductCollection $collection * @return array diff --git a/etc/communication.xml b/etc/communication.xml index 677b07327..bbbd74187 100644 --- a/etc/communication.xml +++ b/etc/communication.xml @@ -38,4 +38,7 @@ + + + diff --git a/etc/di.xml b/etc/di.xml index b44f2aab7..3f85137fd 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -202,6 +202,13 @@ + + + + Nosto\Tagging\Model\Service\Sync\Delete\AsyncBulkConsumer + + + @@ -211,11 +218,19 @@ 60 + + + 100 + + Nosto\Tagging\Model\Service\Sync\Upsert\AsyncBulkPublisher + + Nosto\Tagging\Model\Service\Sync\Delete\AsyncBulkPublisher + 500 diff --git a/etc/queue.xml b/etc/queue.xml index 0c90c5028..95666bc8a 100644 --- a/etc/queue.xml +++ b/etc/queue.xml @@ -41,4 +41,10 @@ consumerInstance="Magento\Framework\MessageQueue\Consumer" handler="Nosto\Tagging\Model\Service\Sync\Upsert\AsyncBulkConsumer::process"/> + + + diff --git a/etc/queue_consumer.xml b/etc/queue_consumer.xml index 15b96bd92..d6c596e3c 100644 --- a/etc/queue_consumer.xml +++ b/etc/queue_consumer.xml @@ -41,4 +41,10 @@ maxMessages="20" consumerInstance="Magento\Framework\MessageQueue\Consumer" handler="Nosto\Tagging\Model\Service\Sync\Upsert\AsyncBulkConsumer::processOperation"/> + diff --git a/etc/queue_publisher.xml b/etc/queue_publisher.xml index 16b0c8f2e..0d044bc71 100644 --- a/etc/queue_publisher.xml +++ b/etc/queue_publisher.xml @@ -38,4 +38,7 @@ + + + diff --git a/etc/queue_topology.xml b/etc/queue_topology.xml index 6fb2b8d10..8f70a8190 100644 --- a/etc/queue_topology.xml +++ b/etc/queue_topology.xml @@ -37,5 +37,6 @@ +