make changes for import/export module to work properly (#4270)

* pass checked document ids to toolbar

* use dismissProgressNotification

* pass notification options and sent results as an event in notif

* remove notification options and use req.body.messages to sent results as an event in notif

* adds the option originalResponse to the http remote method for the backend

* disable export for users

* importExport

* adds new color variable

* allows to pass existing ID to attachment insert method

* when passing attachmentId check if this one exists already

* adds update method

* returns promise in order to wait if needed

* starts notification with percentage of 0 to avoid NaN

* removes an useless return satement

* in attachment update method, make sure to delete the existing retrieved attachment

* fixes lint

* adds changelog

* minor changes

* compute new docIds and archivedDocIds when updating attachment to verify they exist

* moves import export related options to the module itself

* only gets existing related docs from IDs

* typos

* typo

* use docIds and archivedDocIds from existing attachment to get related  docs

---------

Co-authored-by: Jed <[email protected]>

CHANGELOG.md

@@ -11,6 +11,9 @@
 
 * Widget schema can now follow the parent schema via the similar to introduced in the `array` field type syntax (`<` prefix). In order a parent followed field to be available to the widget schema, the area field should follow it. For example, if area follows the root schema `title` field via `following: ['title']`, any field from a widget schema inside that area can do `following: ['<title']`.
 * The values of fields followed by an `area` field are now available in custom widget preview Vue components (registered with widget option `options.widget = 'MyComponentPreview'`). Those components will also receive additional `areaField` prop (the parent area field definition object).
+* Allows to insert attachments with a given ID, as well as with `docIds` and `archivedDocIds` to preserve related docs.
+* Adds an `update` method to the attachment module, that updates the mongoDB doc and the associated file.
+* Adds an option to the `http` `remote` method to allow receiving the original response from `node-fetch` that is a stream.
 
 ## 3.57.0 2023-09-27
 

modules/@apostrophecms/attachment/index.js

@@ -386,8 +386,7 @@ module.exports = {
       // This method returns `attachment` where `attachment` is an attachment
       // object, suitable for passing to the `url` API and for use as the value
       // of a `type: 'attachment'` schema field.
-      async insert(req, file, options) {
-        options = options || {};
+      async insert(req, file, options = {}) {
         let extension = path.extname(file.name);
         if (extension && extension.length) {
           extension = extension.substr(1);
@@ -402,16 +401,21 @@ module.exports = {
             extensions: accepted.join(req.t('apostrophe:listJoiner'))
           }));
         }
+
+        if (options.attachmentId && await self.apos.attachment.db.findOne({ _id: options.attachmentId })) {
+          throw self.apos.error('invalid', 'duplicate');
+        }
+
         const info = {
-          _id: self.apos.util.generateId(),
+          _id: options.attachmentId ?? self.apos.util.generateId(),
           group: group.name,
           createdAt: new Date(),
           name: self.apos.util.slugify(path.basename(file.name, path.extname(file.name))),
           title: self.apos.util.sortify(path.basename(file.name, path.extname(file.name))),
           extension: extension,
           type: 'attachment',
-          docIds: [],
-          archivedDocIds: []
+          docIds: options.docIds ?? [],
+          archivedDocIds: options.archivedDocIds ?? []
         };
         if (!(options.permissions === false)) {
           if (!self.apos.permission.can(req, 'upload-attachment')) {
@@ -432,7 +436,11 @@ module.exports = {
         }
         if (self.isSized(extension)) {
           // For images we correct automatically for common file extension mistakes
-          const result = await Promise.promisify(self.uploadfs.copyImageIn)(file.path, '/attachments/' + info._id + '-' + info.name, { sizes: self.imageSizes });
+          const result = await Promise.promisify(self.uploadfs.copyImageIn)(
+            file.path,
+            '/attachments/' + info._id + '-' + info.name,
+            { sizes: self.imageSizes }
+          );
           info.extension = result.extension;
           info.width = result.width;
           info.height = result.height;
@@ -454,6 +462,51 @@ module.exports = {
         await self.db.insertOne(info);
         return info;
       },
+
+      async update(req, file, attachment) {
+        const existing = await self.db.findOne({ _id: attachment._id });
+        if (!existing) {
+          throw self.apos.error('notfound');
+        }
+
+        const projection = {
+          _id: 1,
+          archived: 1
+        };
+
+        const existingRelatedDocs = await self.apos.doc.db
+          .find({
+            _id: {
+              $in: [
+                ...existing.docIds,
+                ...existing.archivedDocIds,
+                ...attachment.docIds,
+                ...attachment.archivedDocIds
+              ]
+            }
+          }, { projection })
+          .toArray();
+
+        const { docIds, archivedDocIds } = existingRelatedDocs
+          .reduce(({ docIds, archivedDocIds }, doc) => {
+            return {
+              docIds: [ ...docIds, ...!doc.archived ? [ doc._id ] : [] ],
+              archivedDocIds: [ ...archivedDocIds, ...doc.archived ? [ doc._id ] : [] ]
+            };
+          }, {
+            docIds: [],
+            archivedDocIds: []
+          });
+
+        await self.alterAttachment(existing, 'remove');
+        await self.db.deleteOne({ _id: existing._id });
+        await self.insert(req, file, {
+          attachmentId: attachment._id,
+          docIds: _.uniq([ ...docIds, ...existing.docIds || [] ]),
+          archivedDocIds: _.uniq([ ...archivedDocIds, ...existing.archivedDocIds || [] ])
+        });
+      },
+
       // Given a path to a local svg file, sanitize any XSS attack vectors that
       // may be present in the file. The caller is responsible for catching any
       // exception thrown and treating that as an invalid file but there is no

modules/@apostrophecms/http/index.js

@@ -162,6 +162,7 @@ module.exports = {
       // `fullResponse` (if true, return an object with `status`, `headers` and `body`
       // properties, rather than returning the body directly; the individual `headers` are canonicalized
       // to lowercase names. If a header appears multiple times an array is returned for it)
+      // `originalResponse` (if true, return the response object exactly as it is returned by node-fetch)
       //
       // If the status code is >= 400 an error is thrown. The error object will be
       // similar to a `fullResponse` object, with a `status` property.
@@ -225,6 +226,9 @@ module.exports = {
             options.jar.setCookieSync(cookie, url);
           });
         }
+        if (options.originalResponse) {
+          return res;
+        }
         awaitedBody = true;
         body = await getBody();
         if (res.status >= 400) {

modules/@apostrophecms/image/ui/apos/components/AposMediaManager.vue

@@ -56,6 +56,7 @@
             :labels="moduleLabels"
             :disable="relationshipErrors === 'min'"
             :displayed-items="items.length"
+            :checked="checked"
             :checked-count="checked.length"
             :module-name="moduleName"
             @page-change="updatePage"

modules/@apostrophecms/job/index.js

@@ -242,7 +242,7 @@ module.exports = {
             await self.triggerNotification(req, 'completed', {
               count: total,
               dismiss: true
-            });
+            }, results);
             // Dismiss the progress notification. It will delay 4 seconds
             // because "completed" notification will dismiss in 5 and we want
             // to maintain the feeling of process order for users.
@@ -261,11 +261,18 @@ module.exports = {
       //   array
       // No messages are required, but they provide helpful information to
       // end users.
-      async triggerNotification(req, stage, options = {}) {
+      async triggerNotification(req, stage, options = {}, results) {
         if (!req.body || !req.body.messages || !req.body.messages[stage]) {
           return {};
         }
 
+        const event = req.body.messages.resultsEventName && results
+          ? {
+            name: req.body.messages.resultsEventName,
+            data: { ...results }
+          }
+          : null;
+
         return self.apos.notification.trigger(req, req.body.messages[stage], {
           interpolate: {
             count: options.count || (req.body._ids && req.body._ids.length),
@@ -277,6 +284,7 @@ module.exports = {
             action: options.action,
             ids: options.ids
           },
+          event,
           icon: req.body.messages.icon || 'database-export-icon',
           type: options.type || 'success',
           return: true

modules/@apostrophecms/modal/ui/apos/components/AposDocsManagerToolbar.vue

@@ -124,6 +124,10 @@ export default {
       type: Number,
       required: true
     },
+    checked: {
+      type: Array,
+      default: () => []
+    },
     checkedCount: {
       type: Number,
       required: true
@@ -273,7 +277,7 @@ export default {
       modal, ...rest
     }) {
       await apos.modal.execute(modal, {
-        count: this.checkedCount,
+        checked: this.checked,
         moduleName: this.moduleName,
         ...rest
       });

modules/@apostrophecms/notification/ui/apos/components/AposNotification.vue

@@ -71,6 +71,7 @@ export default {
         route: `${apos.modules['@apostrophecms/job'].action}/${this.notification.job._id}`,
         processed: 0,
         total: 1,
+        percentage: 0,
         action: this.notification.job.action
       } : null
     };

modules/@apostrophecms/piece-type/ui/apos/components/AposDocsManager.vue

@@ -70,6 +70,7 @@
             :labels="moduleLabels"
             :displayed-items="items.length"
             :is-relationship="!!relationshipField"
+            :checked="checked"
             :checked-count="checked.length"
             :batch-operations="moduleOptions.batchOperations"
             :module-name="moduleName"

modules/@apostrophecms/ui/ui/apos/scss/global/_theme.scss

@@ -5,6 +5,7 @@
   --a-success: #00bf9a;
   --a-success-fade: #00bf9a30;
   --a-warning: #ffce00;
+  --a-warning-dark: #a75c07;
   --a-warning-fade: #ffce0030;
   --a-progress-bg: #2c354d;