diff --git a/Model/Indexer/ProductIndexer.php b/Model/Indexer/ProductIndexer.php
index c6dc1e725..fedc18803 100644
--- a/Model/Indexer/ProductIndexer.php
+++ b/Model/Indexer/ProductIndexer.php
@@ -128,6 +128,38 @@ public function doIndex(Store $store, array $ids = [])
$collection,
$store
);
+ $this->handleDeletedProducts($collection, $store, $ids);
+ }
+
+ /**
+ * @param ProductCollection $existingCollection
+ * @param Store $store
+ * @param array $givenIds
+ * @throws NostoException
+ */
+ private function handleDeletedProducts(ProductCollection $existingCollection, Store $store, array $givenIds)
+ {
+ if (!empty($givenIds)) {
+ $existingCollection->setPageSize(1000);
+ $iterator = new PagingIterator($existingCollection);
+ $present = [];
+ foreach ($iterator as $page) {
+ foreach ($page->getItems() as $item) {
+ /** @noinspection PhpPossiblePolymorphicInvocationInspection */
+ $id = $item->getId();
+ $present[$id] = $id;
+ }
+ }
+ $removed = [];
+ foreach ($givenIds as $productId) {
+ if (!isset($present[$productId])) {
+ $removed[] = $productId;
+ }
+ }
+ if (count($removed) > 0) {
+ $this->productUpdateService->addIdsToDeleteMessageQueue($removed, $store);
+ }
+ }
}
/**
diff --git a/Model/Service/Sync/Delete/AsyncBulkConsumer.php b/Model/Service/Sync/Delete/AsyncBulkConsumer.php
new file mode 100644
index 000000000..0b067ebb5
--- /dev/null
+++ b/Model/Service/Sync/Delete/AsyncBulkConsumer.php
@@ -0,0 +1,93 @@
+
+ * @copyright 2020 Nosto Solutions Ltd
+ * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause
+ *
+ */
+
+namespace Nosto\Tagging\Model\Service\Sync\Delete;
+
+use Nosto\NostoException;
+use Nosto\Tagging\Model\Service\Sync\AbstractBulkConsumer;
+use Nosto\Tagging\Helper\Scope as NostoHelperScope;
+use Magento\Framework\EntityManager\EntityManager;
+use Magento\Framework\Json\Helper\Data as JsonHelper;
+use Nosto\Tagging\Logger\Logger;
+use Magento\Store\Model\App\Emulation;
+
+class AsyncBulkConsumer extends AbstractBulkConsumer
+{
+ /** @var DeleteService */
+ private DeleteService $deleteService;
+
+ /** @var NostoHelperScope */
+ private NostoHelperScope $nostoHelperScope;
+
+ /**
+ * AsyncBulkConsumer constructor.
+ * @param DeleteService $deleteService
+ * @param NostoHelperScope $nostoHelperScope
+ * @param JsonHelper $jsonHelper
+ * @param EntityManager $entityManager
+ * @param Emulation $storeEmulation
+ * @param Logger $logger
+ */
+ public function __construct(
+ DeleteService $deleteService,
+ NostoHelperScope $nostoHelperScope,
+ JsonHelper $jsonHelper,
+ EntityManager $entityManager,
+ Emulation $storeEmulation,
+ Logger $logger
+ ) {
+ $this->deleteService = $deleteService;
+ $this->nostoHelperScope = $nostoHelperScope;
+ parent::__construct(
+ $logger,
+ $jsonHelper,
+ $entityManager,
+ $storeEmulation
+ );
+ }
+
+ /**
+ * @inheritDoc
+ * @param array $productIds
+ * @param string $storeId
+ * @throws NostoException
+ */
+ public function doOperation(array $productIds, string $storeId)
+ {
+ $store = $this->nostoHelperScope->getStore($storeId);
+ $this->deleteService->delete($productIds, $store);
+ }
+}
diff --git a/Model/Service/Sync/Delete/AsyncBulkPublisher.php b/Model/Service/Sync/Delete/AsyncBulkPublisher.php
new file mode 100644
index 000000000..5198406c7
--- /dev/null
+++ b/Model/Service/Sync/Delete/AsyncBulkPublisher.php
@@ -0,0 +1,77 @@
+
+ * @copyright 2020 Nosto Solutions Ltd
+ * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause
+ *
+ */
+
+namespace Nosto\Tagging\Model\Service\Sync\Delete;
+
+use Nosto\Tagging\Model\Service\Sync\AbstractBulkPublisher;
+
+class AsyncBulkPublisher extends AbstractBulkPublisher
+{
+ public const NOSTO_DELETE_MESSAGE_QUEUE = 'nosto_product_sync.delete';
+ public const BULK_SIZE = 100;
+
+ /**
+ * @inheritDoc
+ */
+ public function getTopicName(): string
+ {
+ return self::NOSTO_DELETE_MESSAGE_QUEUE;
+ }
+
+ /**
+ * @inheritDoc
+ */
+ public function getBulkSize(): int
+ {
+ return self::BULK_SIZE;
+ }
+
+ /**
+ * @inheritDoc
+ */
+ public function getBulkDescription(): string
+ {
+ return sprintf('Delete %d Nosto products', 2);
+ }
+
+ /**
+ * @inheritDoc
+ */
+ public function getMetaData(): string
+ {
+ return 'Delete Nosto products';
+ }
+}
diff --git a/Model/Service/Sync/Delete/DeleteService.php b/Model/Service/Sync/Delete/DeleteService.php
new file mode 100644
index 000000000..aafb1f39a
--- /dev/null
+++ b/Model/Service/Sync/Delete/DeleteService.php
@@ -0,0 +1,134 @@
+
+ * @copyright 2020 Nosto Solutions Ltd
+ * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause
+ *
+ */
+
+namespace Nosto\Tagging\Model\Service\Sync\Delete;
+
+use Exception;
+use Magento\Store\Model\Store;
+use Nosto\Model\Signup\Account as NostoSignupAccount;
+use Nosto\NostoException;
+use Nosto\Operation\DeleteProduct;
+use Nosto\Tagging\Helper\Account as NostoHelperAccount;
+use Nosto\Tagging\Helper\Data as NostoHelperData;
+use Nosto\Tagging\Helper\Url as NostoHelperUrl;
+use Nosto\Tagging\Logger\Logger as NostoLogger;
+use Nosto\Tagging\Model\Service\AbstractService;
+use Nosto\Tagging\Model\Service\Cache\CacheService;
+
+class DeleteService extends AbstractService
+{
+
+ public const BENCHMARK_DELETE_NAME = 'nosto_product_delete';
+ public const BENCHMARK_DELETE_BREAKPOINT = 1;
+ public const PRODUCT_DELETION_BATCH_SIZE = 100;
+
+ /** @var CacheService */
+ private CacheService $cacheService;
+
+ /** @var NostoHelperAccount */
+ private NostoHelperAccount $nostoHelperAccount;
+
+ /** @var NostoHelperUrl */
+ private NostoHelperUrl $nostoHelperUrl;
+
+ /** @var int */
+ private int $deleteBatchSize;
+
+ /**
+ * DeleteService constructor.
+ * @param CacheService $cacheService
+ * @param NostoHelperAccount $nostoHelperAccount
+ * @param NostoHelperData $nostoHelperData
+ * @param NostoHelperUrl $nostoHelperUrl
+ * @param NostoLogger $logger
+ * @param $deleteBatchSize
+ */
+ public function __construct(
+ CacheService $cacheService,
+ NostoHelperAccount $nostoHelperAccount,
+ NostoHelperData $nostoHelperData,
+ NostoHelperUrl $nostoHelperUrl,
+ NostoLogger $logger,
+ $deleteBatchSize
+ ) {
+ $this->cacheService = $cacheService;
+ $this->nostoHelperAccount = $nostoHelperAccount;
+ $this->nostoHelperUrl = $nostoHelperUrl;
+ $this->deleteBatchSize = $deleteBatchSize;
+ parent::__construct($nostoHelperData, $nostoHelperAccount, $logger);
+ }
+
+ /**
+ * Discontinues products in Nosto and removes indexed products from Nosto product index
+ *
+ * @param array $productIds
+ * @param Store $store
+ * @throws NostoException
+ */
+ public function delete(array $productIds, Store $store)
+ {
+ if (count($productIds) === 0) {
+ return;
+ }
+ $account = $this->nostoHelperAccount->findAccount($store);
+ if ($account instanceof NostoSignupAccount === false) {
+ throw new NostoException(sprintf('Store view %s does not have Nosto installed', $store->getName()));
+ }
+ $this->startBenchmark(self::BENCHMARK_DELETE_NAME, self::BENCHMARK_DELETE_BREAKPOINT);
+ $productIdBatches = array_chunk($productIds, $this->deleteBatchSize);
+ $this->logDebugWithStore(
+ sprintf(
+ 'Deleting total of %d products in batches of %d',
+ count($productIds),
+ count($productIdBatches)
+ ),
+ $store
+ );
+ foreach ($productIdBatches as $ids) {
+ try {
+ $op = new DeleteProduct($account, $this->nostoHelperUrl->getActiveDomain($store));
+ $op->setResponseTimeout(30);
+ $op->setProductIds($ids);
+ $op->delete(); // @codingStandardsIgnoreLine
+ $this->cacheService->removeByProductIds($store, $ids);
+ $this->tickBenchmark(self::BENCHMARK_DELETE_NAME);
+ } catch (Exception $e) {
+ $this->getLogger()->exception($e);
+ }
+ }
+ $this->logBenchmarkSummary(self::BENCHMARK_DELETE_NAME, $store);
+ }
+}
diff --git a/Model/Service/Update/ProductUpdateService.php b/Model/Service/Update/ProductUpdateService.php
index 12c6a6946..2e5a4d8ed 100644
--- a/Model/Service/Update/ProductUpdateService.php
+++ b/Model/Service/Update/ProductUpdateService.php
@@ -61,6 +61,9 @@ class ProductUpdateService extends AbstractService
/** @var BulkPublisherInterface */
private BulkPublisherInterface $upsertBulkPublisher;
+ /** @var BulkPublisherInterface */
+ private BulkPublisherInterface $deleteBulkPublisher;
+
/**
* ProductUpdateService constructor.
* @param NostoLogger $logger
@@ -68,6 +71,7 @@ class ProductUpdateService extends AbstractService
* @param NostoAccountHelper $nostoAccountHelper
* @param NostoProductRepository $nostoProductRepository
* @param BulkPublisherInterface $upsertBulkPublisher
+ * @param BulkPublisherInterface $deleteBulkPublisher
* @param int $batchSize
*/
public function __construct(
@@ -76,11 +80,13 @@ public function __construct(
NostoAccountHelper $nostoAccountHelper,
NostoProductRepository $nostoProductRepository,
BulkPublisherInterface $upsertBulkPublisher,
+ BulkPublisherInterface $deleteBulkPublisher,
int $batchSize
) {
parent::__construct($nostoDataHelper, $nostoAccountHelper, $logger);
$this->nostoProductRepository = $nostoProductRepository;
$this->upsertBulkPublisher = $upsertBulkPublisher;
+ $this->deleteBulkPublisher = $deleteBulkPublisher;
$this->batchSize = $batchSize;
}
@@ -117,6 +123,24 @@ public function addCollectionToUpdateMessageQueue(ProductCollection $collection,
}
}
+ /**
+ * Sets the product ids into the delete message queue
+ *
+ * @param $productIds
+ * @param Store $store
+ */
+ public function addIdsToDeleteMessageQueue($productIds, Store $store)
+ {
+ if ($this->getAccountHelper()->findAccount($store) === null) {
+ $this->logDebugWithStore('No nosto account found for the store', $store);
+ return;
+ }
+ $batchedIds = array_chunk($productIds, $this->batchSize);
+ foreach ($batchedIds as $idBatch) {
+ $this->deleteBulkPublisher->execute($store->getId(), $idBatch);
+ }
+ }
+
/**
* @param ProductCollection $collection
* @return array
diff --git a/etc/communication.xml b/etc/communication.xml
index 677b07327..bbbd74187 100644
--- a/etc/communication.xml
+++ b/etc/communication.xml
@@ -38,4 +38,7 @@
+
+
+
diff --git a/etc/di.xml b/etc/di.xml
index b44f2aab7..3f85137fd 100644
--- a/etc/di.xml
+++ b/etc/di.xml
@@ -202,6 +202,13 @@
+
+
+
+ Nosto\Tagging\Model\Service\Sync\Delete\AsyncBulkConsumer
+
+
+
@@ -211,11 +218,19 @@
60
+
+
+ 100
+
+
Nosto\Tagging\Model\Service\Sync\Upsert\AsyncBulkPublisher
+
+ Nosto\Tagging\Model\Service\Sync\Delete\AsyncBulkPublisher
+
500
diff --git a/etc/queue.xml b/etc/queue.xml
index 0c90c5028..95666bc8a 100644
--- a/etc/queue.xml
+++ b/etc/queue.xml
@@ -41,4 +41,10 @@
consumerInstance="Magento\Framework\MessageQueue\Consumer"
handler="Nosto\Tagging\Model\Service\Sync\Upsert\AsyncBulkConsumer::process"/>
+
+
+
diff --git a/etc/queue_consumer.xml b/etc/queue_consumer.xml
index 15b96bd92..d6c596e3c 100644
--- a/etc/queue_consumer.xml
+++ b/etc/queue_consumer.xml
@@ -41,4 +41,10 @@
maxMessages="20"
consumerInstance="Magento\Framework\MessageQueue\Consumer"
handler="Nosto\Tagging\Model\Service\Sync\Upsert\AsyncBulkConsumer::processOperation"/>
+
diff --git a/etc/queue_publisher.xml b/etc/queue_publisher.xml
index 16b0c8f2e..0d044bc71 100644
--- a/etc/queue_publisher.xml
+++ b/etc/queue_publisher.xml
@@ -38,4 +38,7 @@
+
+
+
diff --git a/etc/queue_topology.xml b/etc/queue_topology.xml
index 6fb2b8d10..8f70a8190 100644
--- a/etc/queue_topology.xml
+++ b/etc/queue_topology.xml
@@ -37,5 +37,6 @@
+