From c4f457f309225de3ae190a773487d9e6f5bce7f4 Mon Sep 17 00:00:00 2001 From: Raghu A <125877471+raga-adbe-gh@users.noreply.github.com> Date: Wed, 20 Mar 2024 00:14:48 +0530 Subject: [PATCH] MWPW-128690 Unit Tests (#93) * Update README.md * MWPW-142865: Stage deploy script updated (#91) * .env template for easier setup and onboarding (#90) * Update README.md * .env template for easier setup and onboarding * remove temp change * MWPW-142865: Stage deploy script updated (#89) * Revert temp readme change * MWPW-128690 Unit Tests Unit Test Coverage for action helpers and utilities. Payload handling updated * Minor fixes * Sharepoint reference updated * Rename FgAction --------- Co-authored-by: Sunil Kamat <107644736+sukamat@users.noreply.github.com> Co-authored-by: Raghu A --- actions/appConfig.js | 116 ++-- actions/batch.js | 4 +- actions/batchManager.js | 15 +- actions/config.js | 118 ---- actions/copy/copy.js | 25 +- actions/copy/worker.js | 141 +---- actions/delete/delete.js | 26 +- actions/delete/worker.js | 34 +- actions/{FgAction.js => fgAction.js} | 31 +- actions/fgCopyActionHelper.js | 132 +++++ actions/fgPromoteActionHelper.js | 256 +++++++++ actions/fgStatus.js | 7 +- actions/fgUser.js | 25 +- actions/helixUtils.js | 29 +- actions/maint/maint.js | 31 +- actions/project.js | 128 ++--- actions/promote/createBatch.js | 89 +-- actions/promote/postCopyWorker.js | 80 +-- actions/promote/promote.js | 25 +- actions/promote/triggerNTrack.js | 49 +- actions/promote/worker.js | 134 +---- actions/promoteStatus/promoteStatus.js | 26 +- actions/sharepoint.js | 756 ++++++++++++------------- actions/sharepointAuth.js | 28 +- actions/status/status.js | 15 +- actions/utils.js | 2 +- package-lock.json | 32 ++ package.json | 1 + test/appConfig.test.js | 30 +- test/batch.test.js | 126 +++++ test/batchManager.test.js | 300 ++++++++++ test/fgAction.test.js | 117 ++++ test/fgStatus.test.js | 125 ++++ test/fgUser.test.js | 91 +++ test/helixUtils.test.js | 109 ++++ test/project.test.js | 63 +++ test/sharepoint.test.js | 570 +++++++++++++++++++ test/sharepointAuth.test.js | 92 +++ test/urlinfo.test.js | 52 ++ test/utils.test.js | 269 +++++---- 40 files changed, 2994 insertions(+), 1305 deletions(-) delete mode 100644 actions/config.js rename actions/{FgAction.js => fgAction.js} (91%) create mode 100644 actions/fgCopyActionHelper.js create mode 100644 actions/fgPromoteActionHelper.js create mode 100644 test/batch.test.js create mode 100644 test/batchManager.test.js create mode 100644 test/fgAction.test.js create mode 100644 test/fgStatus.test.js create mode 100644 test/fgUser.test.js create mode 100644 test/helixUtils.test.js create mode 100644 test/project.test.js create mode 100644 test/sharepoint.test.js create mode 100644 test/sharepointAuth.test.js create mode 100644 test/urlinfo.test.js diff --git a/actions/appConfig.js b/actions/appConfig.js index bd727f5..7ec98fc 100644 --- a/actions/appConfig.js +++ b/actions/appConfig.js @@ -20,21 +20,22 @@ const { strToArray, strToBool, getAioLogger } = require('./utils'); const UrlInfo = require('./urlInfo'); // Max activation is 1hrs, set to 2hrs -const MAX_ACTIVATION_TIME = 2 * 60 * 60 * 1000; -const ENV_VAR_ACTIVATION_ID = '__OW_ACTIVATION_ID'; +const GRAPH_API = 'https://graph.microsoft.com/v1.0'; /** * This store the Floodate configs. * Common Configs - Parameters like Batch */ class AppConfig { - // set payload per activation - configMap = { payload: {} }; + constructor(params) { + this.configMap = { payload: {} }; + if (params) { + this.setAppConfig(params); + } + } setAppConfig(params) { - const payload = this.initPayload(); - // Called during action start to cleanup old entries - this.removeOldPayload(); + const payload = this.getPayload(); // These are payload parameters // eslint-disable-next-line no-underscore-dangle @@ -85,37 +86,8 @@ class AppConfig { }; } - // Activation Payload Related - initPayload() { - this.configMap.payload[this.getPayloadKey()] = { - payloadAccessedOn: new Date().getTime() - }; - return this.configMap.payload[this.getPayloadKey()]; - } - - getPayloadKey() { - return process.env[ENV_VAR_ACTIVATION_ID]; - } - getPayload() { - this.configMap.payload[this.getPayloadKey()].payloadAccessedOn = new Date().getTime(); - return this.configMap.payload[this.getPayloadKey()]; - } - - removePayload() { - delete this.configMap.payload[this.getPayloadKey()]; - } - - /** - * Similar to LRU - */ - removeOldPayload() { - const { payload } = this.configMap; - const payloadKeys = Object.keys(payload); - const leastTime = new Date().getTime(); - payloadKeys - .filter((key) => payload[key]?.payloadAccessedOn < leastTime - MAX_ACTIVATION_TIME) - .forEach((key) => delete payload[key]); + return this.configMap.payload; } // Configs related methods @@ -242,6 +214,74 @@ class AppConfig { getUserToken() { return this.getPayload().spToken; } + + getSpConfig() { + if (!this.getUrlInfo().isValid()) { + return undefined; + } + + const config = this.getConfig(); + + // get drive id if available + const { driveId, rootFolder, fgRootFolder } = this.getPayload(); + const drive = driveId ? `/drives/${driveId}` : '/drive'; + + const baseURI = `${config.fgSite}${drive}/root:${rootFolder}`; + const fgBaseURI = `${config.fgSite}${drive}/root:${fgRootFolder}`; + const baseItemsURI = `${config.fgSite}${drive}/items`; + return { + api: { + url: GRAPH_API, + file: { + get: { baseURI, fgBaseURI }, + download: { baseURI: `${config.fgSite}${drive}/items` }, + upload: { + baseURI, + fgBaseURI, + method: 'PUT', + }, + delete: { + baseURI, + fgBaseURI, + method: 'DELETE', + }, + update: { + baseURI, + fgBaseURI, + method: 'PATCH', + }, + createUploadSession: { + baseURI, + fgBaseURI, + method: 'POST', + payload: { '@microsoft.graph.conflictBehavior': 'replace' }, + }, + copy: { + baseURI, + fgBaseURI, + method: 'POST', + payload: { '@microsoft.graph.conflictBehavior': 'replace' }, + }, + }, + directory: { + create: { + baseURI, + fgBaseURI, + method: 'PATCH', + payload: { folder: {} }, + }, + }, + excel: { + get: { baseItemsURI }, + update: { + baseItemsURI, + method: 'POST', + }, + }, + batch: { uri: `${GRAPH_API}/$batch` }, + }, + }; + } } -module.exports = new AppConfig(); +module.exports = AppConfig; diff --git a/actions/batch.js b/actions/batch.js index e567c7e..8f644d2 100644 --- a/actions/batch.js +++ b/actions/batch.js @@ -17,7 +17,6 @@ ************************************************************************* */ const { getAioLogger } = require('./utils'); -const appConfig = require('./appConfig'); const logger = getAioLogger(); @@ -41,7 +40,7 @@ class Batch { this.filesSdk = params.filesSdk; this.instancePath = params.instancePath; this.batchNumber = params?.batchNumber || 1; - this.maxFilesPerBatch = appConfig.getBatchConfig().maxFilesPerBatch; + this.maxFilesPerBatch = params?.maxFilesPerBatch || 200; this.batchPath = `${this.instancePath}/${FOLDER_PREFIX}_${this.batchNumber}`; this.batchInfoFile = `${this.batchPath}/${BATCH_INFO_FILE}`; this.resultsFile = `${this.batchPath}/${RESULTS_FILE}`; @@ -85,7 +84,6 @@ class Batch { if (!this.filesSdk || !this.batchFiles?.length) return; const dataStr = JSON.stringify(this.batchFiles); await this.filesSdk.write(this.batchInfoFile, dataStr); - this.batchInfoFile = []; } async getFiles() { diff --git a/actions/batchManager.js b/actions/batchManager.js index 4e4ef2d..61ea9e3 100644 --- a/actions/batchManager.js +++ b/actions/batchManager.js @@ -17,7 +17,6 @@ ************************************************************************* */ const filesLib = require('@adobe/aio-lib-files'); const Batch = require('./batch'); -const appConfig = require('./appConfig'); const { getAioLogger } = require('./utils'); const logger = getAioLogger(); @@ -37,6 +36,8 @@ const logger = getAioLogger(); * There needs to be enhacement to handle this within this e.g. Batch execution stargergy should be implemented. */ class BatchManager { + batchConfig = null; + filesSdk = null; instanceData = { lastBatch: '', dtls: { batchesInfo: [] } }; @@ -50,7 +51,8 @@ class BatchManager { constructor(params) { this.params = params || {}; this.batches = []; - this.batchFilesPath = appConfig.getBatchConfig()?.batchFilesPath; + this.batchConfig = params.batchConfig; + this.batchFilesPath = this.batchConfig.batchFilesPath; this.key = params.key; this.bmPath = `${this.batchFilesPath}/${this.key}`; this.bmTracker = `${this.bmPath}/tracker.json`; @@ -82,7 +84,8 @@ class BatchManager { ...this.params, filesSdk: this.filesSdk, instancePath: this.instancePath, - batchNumber: this.currentBatchNumber + batchNumber: this.currentBatchNumber, + maxFilesPerBatch: this.batchConfig.maxFilesPerBatch }); this.batches.push(this.currentBatch); } @@ -106,7 +109,8 @@ class BatchManager { * Structure * { * instanceKeys: [_milo_pink], - * '_milo_pink': {done: , proceed: } + * '_milo_pink': {done: , proceed: }, + * * '_bacom_pink': {done: , proceed: }, * } */ async readBmTracker() { @@ -275,7 +279,8 @@ class BatchManager { this.currentBatch = new Batch({ filesSdk: this.filesSdk, instancePath: this.instancePath, - batchNumber: this.currentBatchNumber + batchNumber: this.currentBatchNumber, + maxFilesPerBatch: this.batchConfig.maxFilesPerBatch }); this.batches.push(this.currentBatch); this.instanceData.lastBatch = this.currentBatchNumber; diff --git a/actions/config.js b/actions/config.js deleted file mode 100644 index 95c40f2..0000000 --- a/actions/config.js +++ /dev/null @@ -1,118 +0,0 @@ -/* ************************************************************************ -* ADOBE CONFIDENTIAL -* ___________________ -* -* Copyright 2023 Adobe -* All Rights Reserved. -* -* NOTICE: All information contained herein is, and remains -* the property of Adobe and its suppliers, if any. The intellectual -* and technical concepts contained herein are proprietary to Adobe -* and its suppliers and are protected by all applicable intellectual -* property laws, including trade secret and copyright laws. -* Dissemination of this information or reproduction of this material -* is strictly forbidden unless prior written permission is obtained -* from Adobe. -************************************************************************* */ - -const appConfig = require('./appConfig'); - -const GRAPH_API = 'https://graph.microsoft.com/v1.0'; - -function getSharepointConfig(applicationConfig) { - // get drive id if available - const { driveId } = applicationConfig.payload; - const drive = driveId ? `/drives/${driveId}` : '/drive'; - - const baseURI = `${applicationConfig.fgSite}${drive}/root:${applicationConfig.payload.rootFolder}`; - const fgBaseURI = `${applicationConfig.fgSite}${drive}/root:${applicationConfig.payload.fgRootFolder}`; - const baseItemsURI = `${applicationConfig.fgSite}${drive}/items`; - return { - ...applicationConfig, - clientApp: { - auth: { - clientId: applicationConfig.fgClientId, - authority: applicationConfig.fgAuthority, - }, - cache: { cacheLocation: 'sessionStorage' }, - }, - shareUrl: applicationConfig.shareurl, - fgShareUrl: applicationConfig.fgShareUrl, - login: { redirectUri: '/tools/loc/spauth' }, - api: { - url: GRAPH_API, - file: { - get: { baseURI, fgBaseURI }, - download: { baseURI: `${applicationConfig.fgSite}${drive}/items` }, - upload: { - baseURI, - fgBaseURI, - method: 'PUT', - }, - delete: { - baseURI, - fgBaseURI, - method: 'DELETE', - }, - update: { - baseURI, - fgBaseURI, - method: 'PATCH', - }, - createUploadSession: { - baseURI, - fgBaseURI, - method: 'POST', - payload: { '@microsoft.graph.conflictBehavior': 'replace' }, - }, - copy: { - baseURI, - fgBaseURI, - method: 'POST', - payload: { '@microsoft.graph.conflictBehavior': 'replace' }, - }, - }, - directory: { - create: { - baseURI, - fgBaseURI, - method: 'PATCH', - payload: { folder: {} }, - }, - }, - excel: { - get: { baseItemsURI }, - update: { - baseItemsURI, - method: 'POST', - }, - }, - batch: { uri: `${GRAPH_API}/$batch` }, - }, - }; -} - -function getHelixAdminConfig() { - const adminServerURL = 'https://admin.hlx.page'; - return { - api: { - status: { baseURI: `${adminServerURL}/status` }, - preview: { baseURI: `${adminServerURL}/preview` }, - }, - }; -} - -async function getConfig() { - if (appConfig.getUrlInfo().isValid()) { - const applicationConfig = appConfig.getConfig(); - return { - sp: getSharepointConfig(applicationConfig), - admin: getHelixAdminConfig(), - }; - } - return undefined; -} - -module.exports = { - getConfig, -}; diff --git a/actions/copy/copy.js b/actions/copy/copy.js index 27b8d8d..ffb3a40 100644 --- a/actions/copy/copy.js +++ b/actions/copy/copy.js @@ -17,12 +17,10 @@ // eslint-disable-next-line import/no-extraneous-dependencies const openwhisk = require('openwhisk'); -const { - getAioLogger, COPY_ACTION -} = require('../utils'); +const { getAioLogger, COPY_ACTION } = require('../utils'); const FgStatus = require('../fgStatus'); -const FgAction = require('../FgAction'); -const appConfig = require('../appConfig'); +const FgAction = require('../fgAction'); +const AppConfig = require('../appConfig'); // This returns the activation ID of the action that it called async function main(args) { @@ -37,7 +35,7 @@ async function main(args) { }; const ow = openwhisk(); // Initialize action - const fgAction = new FgAction(COPY_ACTION, args); + const fgAction = new FgAction(COPY_ACTION, new AppConfig(args)); fgAction.init({ ow }); const { fgStatus } = fgAction.getActionParams(); @@ -45,7 +43,7 @@ async function main(args) { // Validations const vStat = await fgAction.validateAction(valParams); if (vStat && vStat.code !== 200) { - return exitAction(vStat); + return vStat; } fgAction.logStart(); @@ -54,7 +52,7 @@ async function main(args) { status: FgStatus.PROJECT_STATUS.STARTED, statusMessage: 'Triggering copy action' }); - return exitAction(ow.actions.invoke({ + return ow.actions.invoke({ name: 'milo-fg/copy-worker', blocking: false, // this is the flag that instructs to execute the worker asynchronous result: false, @@ -80,7 +78,7 @@ async function main(args) { code: 500, payload: respPayload }; - })); + }); } catch (err) { respPayload = fgStatus.updateStatusToStateLib({ status: FgStatus.PROJECT_STATUS.FAILED, @@ -89,15 +87,10 @@ async function main(args) { logger.error(err); } - return exitAction({ + return { code: 500, payload: respPayload, - }); -} - -function exitAction(resp) { - appConfig.removePayload(); - return resp; + }; } exports.main = main; diff --git a/actions/copy/worker.js b/actions/copy/worker.js index 9d43a57..f64f59c 100644 --- a/actions/copy/worker.js +++ b/actions/copy/worker.js @@ -16,21 +16,16 @@ ************************************************************************* */ const openwhisk = require('openwhisk'); -const { getProjectDetails, updateProjectWithDocs } = require('../project'); +const Sharepoint = require('../sharepoint'); +const Project = require('../project'); const { - updateExcelTable, getFile, saveFile, copyFile, bulkCreateFolders -} = require('../sharepoint'); -const { - toUTCStr, getAioLogger, handleExtension, delay, logMemUsage, COPY_ACTION + getAioLogger, logMemUsage, COPY_ACTION } = require('../utils'); -const helixUtils = require('../helixUtils'); +const HelixUtils = require('../helixUtils'); const FgStatus = require('../fgStatus'); -const FgAction = require('../FgAction'); -const sharepointAuth = require('../sharepointAuth'); -const appConfig = require('../appConfig'); - -const BATCH_REQUEST_COPY = 20; -const DELAY_TIME_COPY = 3000; +const FgAction = require('../fgAction'); +const AppConfig = require('../appConfig'); +const FgCopyActionHelper = require('../fgCopyActionHelper'); async function main(params) { logMemUsage(); @@ -42,7 +37,8 @@ async function main(params) { }; const ow = openwhisk(); // Initialize action - const fgAction = new FgAction(COPY_ACTION, params); + const appConfig = new AppConfig(params); + const fgAction = new FgAction(COPY_ACTION, appConfig); fgAction.init({ ow, skipUserDetails: true }); const { fgStatus } = fgAction.getActionParams(); const { projectExcelPath, fgColor } = appConfig.getPayload(); @@ -50,7 +46,7 @@ async function main(params) { // Validations const vStat = await fgAction.validateAction(valParams); if (vStat && vStat.code !== 200) { - return exitAction(vStat); + return vStat; } respPayload = 'Getting all files to be floodgated from the project excel file'; @@ -60,7 +56,11 @@ async function main(params) { statusMessage: respPayload }); - const projectDetail = await getProjectDetails(projectExcelPath); + const sharepoint = new Sharepoint(appConfig); + const project = new Project({ sharepoint }); + const fgCopyActionHelper = new FgCopyActionHelper(); + const helixUtils = new HelixUtils(appConfig); + const projectDetail = await project.getProjectDetails(projectExcelPath); respPayload = 'Injecting sharepoint data'; logger.info(respPayload); @@ -68,7 +68,7 @@ async function main(params) { status: FgStatus.PROJECT_STATUS.IN_PROGRESS, statusMessage: respPayload }); - await updateProjectWithDocs(projectDetail); + await project.updateProjectWithDocs(projectDetail); respPayload = 'Start floodgating content'; logger.info(respPayload); @@ -77,7 +77,7 @@ async function main(params) { statusMessage: respPayload }); - respPayload = await floodgateContent(projectExcelPath, projectDetail, fgStatus, fgColor); + respPayload = await fgCopyActionHelper.floodgateContent(projectExcelPath, projectDetail, fgStatus, fgColor, { sharepoint, helixUtils }); } catch (err) { await fgStatus.updateStatusToStateLib({ status: FgStatus.PROJECT_STATUS.COMPLETED_WITH_ERROR, @@ -87,112 +87,9 @@ async function main(params) { respPayload = err; } logMemUsage(); - return exitAction({ + return { body: respPayload, - }); -} - -async function floodgateContent(projectExcelPath, projectDetail, fgStatus, fgColor) { - const logger = getAioLogger(); - logger.info('Floodgating content started.'); - - async function copyFilesToFloodgateTree(fileInfo) { - const status = { success: false }; - if (!fileInfo?.doc) return status; - - try { - const srcPath = fileInfo.doc.filePath; - logger.info(`Copying ${srcPath} to floodgated folder`); - - let copySuccess = false; - const destinationFolder = `${srcPath.substring(0, srcPath.lastIndexOf('/'))}`; - copySuccess = await copyFile(srcPath, destinationFolder, undefined, true); - if (copySuccess === false) { - logger.info(`Copy was not successful for ${srcPath}. Alternate copy option will be used`); - const file = await getFile(fileInfo.doc); - if (file) { - const destination = fileInfo.doc.filePath; - if (destination) { - // Save the file in the floodgate destination location - const saveStatus = await saveFile(file, destination, true); - if (saveStatus.success) { - copySuccess = true; - } - } - } - } - status.success = copySuccess; - status.srcPath = srcPath; - status.url = fileInfo.doc.url; - } catch (error) { - logger.error(`Error occurred when trying to copy files to floodgated content folder ${error.message}`); - } - return status; - } - - // create batches to process the data - const contentToFloodgate = [...projectDetail.urls]; - const batchArray = []; - for (let i = 0; i < contentToFloodgate.length; i += BATCH_REQUEST_COPY) { - const arrayChunk = contentToFloodgate.slice(i, i + BATCH_REQUEST_COPY); - batchArray.push(arrayChunk); - } - - // process data in batches - const copyStatuses = []; - // Get the access token to cache, avoidid parallel hits to this in below loop. - await sharepointAuth.getAccessToken(); - for (let i = 0; i < batchArray.length; i += 1) { - // Log memory usage per batch as copy is a heavy operation. Can be removed after testing are done. - // Can be replaced with logMemUsageIter for regular logging - logMemUsage(); - logger.info(`Batch create folder ${i} in progress`); - // eslint-disable-next-line no-await-in-loop - await bulkCreateFolders(batchArray[i], true); - logger.info(`Batch copy ${i} in progress`); - // eslint-disable-next-line no-await-in-loop - copyStatuses.push(...await Promise.all( - batchArray[i].map((files) => copyFilesToFloodgateTree(files[1])), - )); - logger.info(`Batch copy ${i} completed`); - // eslint-disable-next-line no-await-in-loop, no-promise-executor-return - await delay(DELAY_TIME_COPY); - } - logger.info('Completed floodgating documents listed in the project excel'); - - logger.info('Previewing floodgated files... '); - let previewStatuses = []; - if (helixUtils.canBulkPreviewPublish(true, fgColor)) { - const paths = copyStatuses.filter((ps) => ps.success).map((ps) => handleExtension(ps.srcPath)); - previewStatuses = await helixUtils.bulkPreviewPublish(paths, helixUtils.getOperations().PREVIEW, { isFloodgate: true, fgColor }); - } - logger.info('Completed generating Preview for floodgated files.'); - const failedCopies = copyStatuses.filter((status) => !status.success) - .map((status) => status.srcPath || 'Path Info Not available'); - const failedPreviews = previewStatuses.filter((status) => !status.success) - .map((status) => status.path); - const fgErrors = failedCopies.length > 0 || failedPreviews.length > 0; - const payload = fgErrors ? - 'Error occurred when floodgating content. Check project excel sheet for additional information.' : - 'All tasks for Floodgate Copy completed'; - let status = fgErrors ? FgStatus.PROJECT_STATUS.COMPLETED_WITH_ERROR : FgStatus.PROJECT_STATUS.COMPLETED; - status = fgErrors && failedCopies.length === copyStatuses.length ? FgStatus.PROJECT_STATUS.FAILED : status; - await fgStatus.updateStatusToStateLib({ - status, - statusMessage: payload - }); - - const { startTime: startCopy, endTime: endCopy } = fgStatus.getStartEndTime(); - const excelValues = [['COPY', toUTCStr(startCopy), toUTCStr(endCopy), failedCopies.join('\n'), failedPreviews.join('\n')]]; - await updateExcelTable(projectExcelPath, 'COPY_STATUS', excelValues); - logger.info('Project excel file updated with copy status.'); - - return payload; -} - -function exitAction(resp) { - appConfig.removePayload(); - return resp; + }; } exports.main = main; diff --git a/actions/delete/delete.js b/actions/delete/delete.js index 60822e2..c1af728 100644 --- a/actions/delete/delete.js +++ b/actions/delete/delete.js @@ -17,12 +17,10 @@ // eslint-disable-next-line import/no-extraneous-dependencies const openwhisk = require('openwhisk'); -const { - getAioLogger, DELETE_ACTION -} = require('../utils'); +const { getAioLogger, DELETE_ACTION } = require('../utils'); const FgStatus = require('../fgStatus'); -const FgAction = require('../FgAction'); -const appConfig = require('../appConfig'); +const FgAction = require('../fgAction'); +const AppConfig = require('../appConfig'); // This returns the activation ID of the action that it called async function main(args) { @@ -38,22 +36,23 @@ async function main(args) { }; const ow = openwhisk(); // Initialize action - const fgAction = new FgAction(DELETE_ACTION, args); + const fgAction = new FgAction(DELETE_ACTION, new AppConfig(args)); fgAction.init({ ow }); const { fgStatus } = fgAction.getActionParams(); try { // Validations const vStat = await fgAction.validateAction(valParams); if (vStat && vStat.code !== 200) { - return exitAction(vStat); + return vStat; } + fgAction.logStart(); respPayload = await fgStatus.updateStatusToStateLib({ status: FgStatus.PROJECT_STATUS.STARTED, statusMessage: 'Triggering delete action' }); - return exitAction(ow.actions.invoke({ + return ow.actions.invoke({ name: 'milo-fg/delete-worker', blocking: false, // this is the flag that instructs to execute the worker asynchronous result: false, @@ -79,7 +78,7 @@ async function main(args) { code: 500, payload: respPayload }; - })); + }); } catch (err) { respPayload = fgStatus.updateStatusToStateLib({ status: FgStatus.PROJECT_STATUS.FAILED, @@ -88,15 +87,10 @@ async function main(args) { logger.error(err); } - return exitAction({ + return { code: 500, payload: respPayload, - }); -} - -function exitAction(resp) { - appConfig.removePayload(); - return resp; + }; } exports.main = main; diff --git a/actions/delete/worker.js b/actions/delete/worker.js index 6a846fe..212807f 100644 --- a/actions/delete/worker.js +++ b/actions/delete/worker.js @@ -15,13 +15,13 @@ * from Adobe. ************************************************************************* */ const openwhisk = require('openwhisk'); -const { deleteFloodgateDir, updateExcelTable } = require('../sharepoint'); +const Sharepoint = require('../sharepoint'); const { toUTCStr, getAioLogger, logMemUsage, DELETE_ACTION } = require('../utils'); const FgStatus = require('../fgStatus'); -const FgAction = require('../FgAction'); -const appConfig = require('../appConfig'); +const FgAction = require('../fgAction'); +const AppConfig = require('../appConfig'); async function main(params) { logMemUsage(); @@ -33,53 +33,59 @@ async function main(params) { actParams: ['adminPageUri'], }; const ow = openwhisk(); + // Initialize action - const fgAction = new FgAction(DELETE_ACTION, params); + const appConfig = new AppConfig(params); + const fgAction = new FgAction(DELETE_ACTION, appConfig); fgAction.init({ ow, skipUserDetails: true }); const { fgStatus } = fgAction.getActionParams(); const { projectExcelPath } = appConfig.getPayload(); + try { // Validations const vStat = await fgAction.validateAction(valParams); if (vStat && vStat.code !== 200) { - return exitAction(vStat); + return vStat; } + respPayload = 'Started deleting content'; logger.info(respPayload); + await fgStatus.updateStatusToStateLib({ status: FgStatus.PROJECT_STATUS.IN_PROGRESS, statusMessage: respPayload }); - const deleteStatus = await deleteFloodgateDir(); + const sharepoint = new Sharepoint(appConfig); + const deleteStatus = await sharepoint.deleteFloodgateDir(); respPayload = deleteStatus === false ? 'Error occurred when deleting content. Check project excel sheet for additional information.' : 'Delete action was completed'; + await fgStatus.updateStatusToStateLib({ status: deleteStatus === false ? FgStatus.PROJECT_STATUS.COMPLETED_WITH_ERROR : FgStatus.PROJECT_STATUS.COMPLETED, statusMessage: respPayload }); + const { startTime: startDelete, endTime: endDelete } = fgStatus.getStartEndTime(); const excelValues = [['DELETE', toUTCStr(startDelete), toUTCStr(endDelete), respPayload]]; - await updateExcelTable(projectExcelPath, 'DELETE_STATUS', excelValues); + + await sharepoint.updateExcelTable(projectExcelPath, 'DELETE_STATUS', excelValues); logger.info('Project excel file updated with delete status.'); } catch (err) { await fgStatus.updateStatusToStateLib({ status: FgStatus.PROJECT_STATUS.COMPLETED_WITH_ERROR, statusMessage: err.message }); + logger.error(err); respPayload = err; } + logMemUsage(); - return exitAction({ + return { body: respPayload, - }); -} - -function exitAction(resp) { - appConfig.removePayload(); - return resp; + }; } exports.main = main; diff --git a/actions/FgAction.js b/actions/fgAction.js similarity index 91% rename from actions/FgAction.js rename to actions/fgAction.js index 246d874..ce69c15 100644 --- a/actions/FgAction.js +++ b/actions/fgAction.js @@ -15,8 +15,12 @@ * from Adobe. ************************************************************************* */ const openwhisk = require('openwhisk'); -const { getAioLogger, actInProgress, DELETE_ACTION, PROMOTE_ACTION } = require('./utils'); -const appConfig = require('./appConfig'); +const { + getAioLogger, + actInProgress, + DELETE_ACTION, + PROMOTE_ACTION +} = require('./utils'); const FgUser = require('./fgUser'); const FgStatus = require('./fgStatus'); @@ -32,18 +36,19 @@ const ALL_OK_SC = 200; * The common parameter validation, user check, */ class FgAction { - constructor(action, params) { + appConfig = null; + + constructor(action, appConfig) { this.action = action || FG_PROOCESS_ACTION; - appConfig.setAppConfig(params); - this.spToken = appConfig.getUserToken(); + this.appConfig = appConfig; // Defaults this.fgUser = null; } init({ fgStatusParams, skipUserDetails = false, ow }) { - const statsParams = { action: this.action, ...fgStatusParams }; + const statsParams = { action: this.action, appConfig: this.appConfig, ...fgStatusParams }; if (!skipUserDetails) { - this.fgUser = new FgUser({ at: this.spToken }); + this.fgUser = new FgUser({ appConfig: this.appConfig }); statsParams.userDetails = this.fgUser.getUserDetails(); } this.fgStatus = new FgStatus(statsParams); @@ -53,7 +58,7 @@ class FgAction { getActionParams() { return { action: this.action, - appConfig, + appConfig: this.appConfig, fgStatus: this.fgStatus, fgUser: this.fgUser }; @@ -67,7 +72,7 @@ class FgAction { async validateStatusParams(statParams = []) { const resp = { ok: false, message: 'Status Params Validation' }; logger.debug(resp.message); - const conf = appConfig.getPayload(); + const conf = this.appConfig.getPayload(); const valFailed = statParams.find((p) => !conf[p]) !== undefined; if (valFailed) { resp.message = 'Could not determine the project path. Try reloading the page and trigger the action again.'; @@ -87,7 +92,7 @@ class FgAction { const resp = { ok: false, message: 'Params Validation.' }; logger.debug(resp.message); let stepMsg; - const conf = appConfig.getPayload(); + const conf = this.appConfig.getPayload(); const valFailed = reqParams.find((p) => !conf[p]) !== undefined; if (valFailed) { stepMsg = `Required data is not available to proceed with FG ${this.action}.`; @@ -103,8 +108,8 @@ class FgAction { } isActionEnabled() { - return (this.action === PROMOTE_ACTION && appConfig.getEnablePromote()) || - (this.action === DELETE_ACTION && appConfig.getEnableDelete()); + return (this.action === PROMOTE_ACTION && this.appConfig.getEnablePromote()) || + (this.action === DELETE_ACTION && this.appConfig.getEnableDelete()); } /** @@ -163,7 +168,7 @@ class FgAction { const actId = storeValue?.action?.activationId; const fgInProg = FgStatus.isInProgress(svStatus); - if (!appConfig.getSkipInProgressCheck() && fgInProg) { + if (!this.appConfig.getSkipInProgressCheck() && fgInProg) { if (!checkActivation || await actInProgress(this.ow, actId, FgStatus.isInProgress(svStatus))) { stepMsg = `A ${this.action} project with activationid: ${storeValue?.action?.activationId} is already in progress. Not triggering this action. And the previous action can be retrieved by refreshing the console page`; diff --git a/actions/fgCopyActionHelper.js b/actions/fgCopyActionHelper.js new file mode 100644 index 0000000..d0f5478 --- /dev/null +++ b/actions/fgCopyActionHelper.js @@ -0,0 +1,132 @@ +/* *********************************************************************** + * ADOBE CONFIDENTIAL + * ___________________ + * + * Copyright 2023 Adobe + * All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of Adobe and its suppliers, if any. The intellectual + * and technical concepts contained herein are proprietary to Adobe + * and its suppliers and are protected by all applicable intellectual + * property laws, including trade secret and copyright laws. + * Dissemination of this information or reproduction of this material + * is strictly forbidden unless prior written permission is obtained + * from Adobe. + ************************************************************************* */ +const { + handleExtension, + toUTCStr, + delay, + getAioLogger, + logMemUsage +} = require('./utils'); +const FgStatus = require('./fgStatus'); + +const BATCH_REQUEST_COPY = 20; +const DELAY_TIME_COPY = 3000; + +/** + * Floodgate action helper routines + */ +class FloodgateActionHelper { + async floodgateContent(projectExcelPath, projectDetail, fgStatus, fgColor, { sharepoint, helixUtils }) { + const logger = getAioLogger(); + logger.info('Floodgating content started.'); + + async function copyFilesToFloodgateTree(fileInfo) { + const status = { success: false }; + if (!fileInfo?.doc) return status; + + try { + const srcPath = fileInfo.doc.filePath; + logger.info(`Copying ${srcPath} to floodgated folder`); + + let copySuccess = false; + const destinationFolder = `${srcPath.substring(0, srcPath.lastIndexOf('/'))}`; + copySuccess = await sharepoint.copyFile(srcPath, destinationFolder, undefined, true); + if (copySuccess === false) { + logger.info(`Copy was not successful for ${srcPath}. Alternate copy option will be used`); + const file = await sharepoint.getFile(fileInfo.doc); + if (file) { + const destination = fileInfo.doc.filePath; + if (destination) { + // Save the file in the floodgate destination location + const saveStatus = await sharepoint.saveFile(file, destination, true); + if (saveStatus.success) { + copySuccess = true; + } + } + } + } + status.success = copySuccess; + status.srcPath = srcPath; + status.url = fileInfo.doc.url; + } catch (error) { + logger.error(`Error occurred when trying to copy files to floodgated content folder ${error.message}`); + } + return status; + } + + // create batches to process the data + const contentToFloodgate = [...projectDetail.urls]; + const batchArray = []; + for (let i = 0; i < contentToFloodgate.length; i += BATCH_REQUEST_COPY) { + const arrayChunk = contentToFloodgate.slice(i, i + BATCH_REQUEST_COPY); + batchArray.push(arrayChunk); + } + + // process data in batches + const copyStatuses = []; + // Get the access token to cache, avoidid parallel hits to this in below loop. + await sharepoint.getSharepointAuth().getAccessToken(); + for (let i = 0; i < batchArray.length; i += 1) { + // Log memory usage per batch as copy is a heavy operation. Can be removed after testing are done. + // Can be replaced with logMemUsageIter for regular logging + logMemUsage(); + logger.info(`Batch create folder ${i} in progress`); + // eslint-disable-next-line no-await-in-loop + await sharepoint.bulkCreateFolders(batchArray[i], true); + logger.info(`Batch copy ${i} in progress`); + // eslint-disable-next-line no-await-in-loop + copyStatuses.push(...await Promise.all( + batchArray[i].map((files) => copyFilesToFloodgateTree(files[1])), + )); + logger.info(`Batch copy ${i} completed`); + // eslint-disable-next-line no-await-in-loop, no-promise-executor-return + await delay(DELAY_TIME_COPY); + } + logger.info('Completed floodgating documents listed in the project excel'); + + logger.info('Previewing floodgated files... '); + let previewStatuses = []; + if (helixUtils.canBulkPreviewPublish(true, fgColor)) { + const paths = copyStatuses.filter((ps) => ps.success).map((ps) => handleExtension(ps.srcPath)); + previewStatuses = await helixUtils.bulkPreviewPublish(paths, helixUtils.getOperations().PREVIEW, { isFloodgate: true, fgColor }); + } + logger.info('Completed generating Preview for floodgated files.'); + const failedCopies = copyStatuses.filter((status) => !status.success) + .map((status) => status.srcPath || 'Path Info Not available'); + const failedPreviews = previewStatuses.filter((status) => !status.success) + .map((status) => status.path); + const fgErrors = failedCopies.length > 0 || failedPreviews.length > 0; + const payload = fgErrors ? + 'Error occurred when floodgating content. Check project excel sheet for additional information.' : + 'All tasks for Floodgate Copy completed'; + let status = fgErrors ? FgStatus.PROJECT_STATUS.COMPLETED_WITH_ERROR : FgStatus.PROJECT_STATUS.COMPLETED; + status = fgErrors && failedCopies.length === copyStatuses.length ? FgStatus.PROJECT_STATUS.FAILED : status; + await fgStatus.updateStatusToStateLib({ + status, + statusMessage: payload + }); + + const { startTime: startCopy, endTime: endCopy } = fgStatus.getStartEndTime(); + const excelValues = [['COPY', toUTCStr(startCopy), toUTCStr(endCopy), failedCopies.join('\n'), failedPreviews.join('\n')]]; + await sharepoint.updateExcelTable(projectExcelPath, 'COPY_STATUS', excelValues); + logger.info('Project excel file updated with copy status.'); + + return payload; + } +} + +module.exports = FloodgateActionHelper; diff --git a/actions/fgPromoteActionHelper.js b/actions/fgPromoteActionHelper.js new file mode 100644 index 0000000..cc1b78c --- /dev/null +++ b/actions/fgPromoteActionHelper.js @@ -0,0 +1,256 @@ +/* *********************************************************************** + * ADOBE CONFIDENTIAL + * ___________________ + * + * Copyright 2023 Adobe + * All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of Adobe and its suppliers, if any. The intellectual + * and technical concepts contained herein are proprietary to Adobe + * and its suppliers and are protected by all applicable intellectual + * property laws, including trade secret and copyright laws. + * Dissemination of this information or reproduction of this material + * is strictly forbidden unless prior written permission is obtained + * from Adobe. + ************************************************************************* */ +const { + handleExtension, + delay, + isFilePatternMatched, + getAioLogger, + logMemUsage +} = require('./utils'); +const Sharepoint = require('./sharepoint'); + +const DELAY_TIME_PROMOTE = 3000; +const MAX_CHILDREN = 5000; + +class FgPromoteActionHelper { + /** + * Find all files in the FG tree to promote. Add to batches. + * @param {BatchManager} batchManager - Instead of BatchManager + * @param {AppConfig} appConfig - Application config with payload + * @returns N/A + */ + async createBatch(batchManager, appConfig) { + const logger = getAioLogger(); + const sp = appConfig.getSpConfig(); + const sharepoint = new Sharepoint(appConfig); + const options = await sharepoint.getAuthorizedRequestOption({ method: 'GET' }); + const promoteIgnoreList = appConfig.getPromoteIgnorePaths(); + logger.info(`Promote ignore list: ${promoteIgnoreList}`); + + // Temporarily restricting the iteration for promote to under /drafts folder only + return this.findAndBatchFGFiles({ + baseURI: sp.api.file.get.fgBaseURI, + options, + fgFolders: (appConfig || appConfig.isDraftOnly()) ? ['/drafts'] : [''], + promoteIgnoreList, + downloadBaseURI: sp.api.file.download.baseURI, + sharepoint + }, batchManager); + } + + /** + * Iteratively finds all files under a specified root folder. Add them to batches + */ + async findAndBatchFGFiles( + { + baseURI, options, fgFolders, promoteIgnoreList, downloadBaseURI, sharepoint + }, batchManager + ) { + const logger = getAioLogger(); + const fgRoot = baseURI.split(':').pop(); + const pPathRegExp = new RegExp(`.*:${fgRoot}`); + while (fgFolders.length !== 0) { + const uri = `${baseURI}${fgFolders.shift()}:/children?$top=${MAX_CHILDREN}`; + // eslint-disable-next-line no-await-in-loop + const res = await sharepoint.fetchWithRetry(uri, options); + if (res.ok) { + // eslint-disable-next-line no-await-in-loop + const json = await res.json(); + // eslint-disable-next-line no-await-in-loop + const driveItems = json.value; + for (let di = 0; di < driveItems?.length; di += 1) { + const item = driveItems[di]; + const itemPath = `${item.parentReference.path.replace(pPathRegExp, '')}/${item.name}`; + if (!isFilePatternMatched(itemPath, promoteIgnoreList)) { + if (item.folder) { + // it is a folder + fgFolders.push(itemPath); + } else { + const downloadUrl = `${downloadBaseURI}/${item.id}/content`; + // eslint-disable-next-line no-await-in-loop + await batchManager.addFile({ fileDownloadUrl: downloadUrl, filePath: itemPath }); + } + } else { + logger.info(`Ignored from promote: ${itemPath}`); + } + } + } + } + } + + /** + * Copies the Floodgated files back to the main content tree. + * Creates intermediate folders if needed. + */ + async promoteCopy(srcPath, destinationFolder, { sharepoint, sp }) { + const { baseURI } = sp.api.file.copy; + const rootFolder = baseURI.split('/').pop(); + const payload = { ...sp.api.file.copy.payload, parentReference: { path: `${rootFolder}${destinationFolder}` } }; + const options = await sharepoint.getAuthorizedRequestOption({ + method: sp.api.file.copy.method, + body: JSON.stringify(payload), + }); + + // copy source is the pink directory for promote + const copyStatusInfo = await sharepoint.fetchWithRetry(`${sp.api.file.copy.fgBaseURI}${srcPath}:/copy?@microsoft.graph.conflictBehavior=replace`, options); + const statusUrl = copyStatusInfo.headers.get('Location'); + let copySuccess = false; + let copyStatusJson = {}; + while (statusUrl && !copySuccess && copyStatusJson.status !== 'failed') { + // eslint-disable-next-line no-await-in-loop + const status = await sharepoint.fetchWithRetry(statusUrl); + if (status.ok) { + // eslint-disable-next-line no-await-in-loop + copyStatusJson = await status.json(); + copySuccess = copyStatusJson.status === 'completed'; + } + } + return copySuccess; + } + + async promoteFloodgatedFiles(batchManager, appConfig) { + const logger = getAioLogger(); + const sharepoint = new Sharepoint(appConfig); + const sp = await appConfig.getSpConfig(); + // Pre check Access Token + await sharepoint.getSharepointAuth().getAccessToken(); + const { promoteCopy } = this; + + async function promoteFile(batchItem) { + const { fileDownloadUrl, filePath } = batchItem.file; + const status = { success: false, srcPath: filePath }; + try { + let promoteSuccess = false; + const destinationFolder = `${filePath.substring(0, filePath.lastIndexOf('/'))}`; + const copyFileStatus = await promoteCopy(filePath, destinationFolder, { sharepoint, sp }); + if (copyFileStatus) { + promoteSuccess = true; + } else { + const file = await sharepoint.getFileUsingDownloadUrl(fileDownloadUrl); + const saveStatus = await sharepoint.saveFile(file, filePath); + if (saveStatus.success) { + promoteSuccess = true; + } + } + status.success = promoteSuccess; + } catch (error) { + const errorMessage = `Error promoting files ${fileDownloadUrl} at ${filePath} to main content tree ${error.message}`; + logger.error(errorMessage); + status.success = false; + } + return status; + } + + let i = 0; + let stepMsg = 'Getting all floodgated files to promote.'; + // Get the batch files using the batchmanager for the assigned batch and process them + const currentBatch = await batchManager.getCurrentBatch(); + const currBatchLbl = `Batch-${currentBatch.getBatchNumber()}`; + const allFloodgatedFiles = await currentBatch?.getFiles(); + logger.info(`Files for the batch are ${allFloodgatedFiles.length}`); + // create batches to process the data + const batchArray = []; + const numBulkReq = appConfig.getNumBulkReq(); + for (i = 0; i < allFloodgatedFiles.length; i += numBulkReq) { + const arrayChunk = allFloodgatedFiles.slice(i, i + numBulkReq); + batchArray.push(arrayChunk); + } + + // process data in batches + const promoteStatuses = []; + for (i = 0; i < batchArray.length; i += 1) { + // eslint-disable-next-line no-await-in-loop + promoteStatuses.push(...await Promise.all( + batchArray[i].map((bi) => promoteFile(bi)) + )); + // eslint-disable-next-line no-await-in-loop, no-promise-executor-return + await delay(DELAY_TIME_PROMOTE); + } + + stepMsg = `Completed promoting all documents in the batch ${currBatchLbl}`; + logger.info(stepMsg); + + const failedPromotes = promoteStatuses.filter((status) => !status.success) + .map((status) => status.srcPath || 'Path Info Not available'); + logger.info(`Promote ${currBatchLbl}, Prm: ${failedPromotes?.length}`); + + if (failedPromotes.length > 0) { + stepMsg = 'Error occurred when promoting floodgated content. Check project excel sheet for additional information.'; + logger.info(stepMsg); + // Write the information to batch manifest + await currentBatch.writeResults({ failedPromotes }); + } else { + stepMsg = `Promoted floodgate for ${currBatchLbl} successfully`; + logger.info(stepMsg); + } + logMemUsage(); + stepMsg = `Floodgate promote (copy) of ${currBatchLbl} is completed`; + return stepMsg; + } + + async previewPublish(doPublish, { batchManager, helixUtils }) { + const logger = getAioLogger(); + + let stepMsg = 'Getting all batch files.'; + // Get the batch files using the batchmanager for the assigned batch and process them + const currentBatch = await batchManager.getCurrentBatch(); + const currBatchLbl = `Batch-${currentBatch.getBatchNumber()}`; + const allFloodgatedFiles = await currentBatch.getFiles(); + const promotedFiles = allFloodgatedFiles.map((e) => e.file.filePath); + const resultsContent = await currentBatch.getResultsContent() || {}; + const failedPromotes = resultsContent.failedPromotes || []; + const prevPaths = promotedFiles.filter((item) => !failedPromotes.includes(item)).map((e) => handleExtension(e)); + logger.info(`Post promote files for ${currBatchLbl} are ${prevPaths?.length}`); + + logger.info('Previewing promoted files.'); + let previewStatuses = []; + let publishStatuses = []; + if (helixUtils.canBulkPreviewPublish()) { + previewStatuses = await helixUtils.bulkPreviewPublish(prevPaths, helixUtils.getOperations().PREVIEW); + stepMsg = 'Completed generating Preview for promoted files.'; + logger.info(stepMsg); + + if (doPublish) { + stepMsg = 'Publishing promoted files.'; + logger.info(stepMsg); + publishStatuses = await helixUtils.bulkPreviewPublish(prevPaths, helixUtils.getOperations().LIVE); + stepMsg = 'Completed Publishing for promoted files'; + logger.info(stepMsg); + } + } + + const failedPreviews = previewStatuses.filter((status) => !status.success) + .map((status) => status.path); + const failedPublishes = publishStatuses.filter((status) => !status.success) + .map((status) => status.path); + logger.info(`Post promote ${currBatchLbl}, Prm: ${failedPromotes?.length}, Prv: ${failedPreviews?.length}, Pub: ${failedPublishes?.length}`); + + if (failedPromotes.length > 0 || failedPreviews.length > 0 || failedPublishes.length > 0) { + stepMsg = 'Error occurred when promoting floodgated content. Check project excel sheet for additional information.'; + logger.info(stepMsg); + // Write the information to batch manifest + currentBatch.writeResults({ failedPromotes, failedPreviews, failedPublishes }); + throw new Error(stepMsg); + } + logMemUsage(); + logger.info(`All tasks for promote ${currBatchLbl} is completed`); + stepMsg = 'All tasks for floodgate promote is completed'; + return stepMsg; + } +} + +module.exports = FgPromoteActionHelper; diff --git a/actions/fgStatus.js b/actions/fgStatus.js index 03af888..081ebcf 100644 --- a/actions/fgStatus.js +++ b/actions/fgStatus.js @@ -18,7 +18,6 @@ const stateLib = require('@adobe/aio-lib-state'); const crypto = require('crypto'); const { getAioLogger, COPY_ACTION, DELETE_ACTION } = require('./utils'); -const appConfig = require('./appConfig'); const FG_KEY = 'FLOODGATE'; @@ -74,8 +73,10 @@ class FgStatus { action, statusKey, keySuffix, + appConfig, userDetails }) { + this.appConfig = appConfig; this.lastTriggeredBy = userDetails?.oid; this.action = action || ''; this.storeKey = statusKey || this.generateStoreKey(keySuffix) || FG_KEY; @@ -83,8 +84,8 @@ class FgStatus { } generateStoreKey(keySuffix) { - const siteFgRootPath = appConfig.getSiteFgRootPath(); - const { projectExcelPath } = appConfig.getPayload(); + const siteFgRootPath = this.appConfig.getSiteFgRootPath(); + const { projectExcelPath } = this.appConfig.getPayload(); let resp = ''; switch (this.action) { diff --git a/actions/fgUser.js b/actions/fgUser.js index fa513fb..86ae71e 100644 --- a/actions/fgUser.js +++ b/actions/fgUser.js @@ -17,17 +17,18 @@ const fetch = require('node-fetch'); const { getAioLogger } = require('./utils'); -const appConfig = require('./appConfig'); -const sharepoint = require('./sharepoint'); -const sharepointAuth = require('./sharepointAuth'); +const Sharepoint = require('./sharepoint'); const logger = getAioLogger(); class FgUser { userGroupIds = []; - constructor({ at }) { - this.at = at; - this.userDetails = sharepointAuth.getUserDetails(at); + constructor({ appConfig }) { + this.appConfig = appConfig; + this.at = this.appConfig.getUserToken(); + this.sharepoint = new Sharepoint(this.appConfig); + this.sharepointAuth = this.sharepoint.getSharepointAuth(); + this.userDetails = this.sharepointAuth.getUserDetails(this.at); this.userOid = this.userDetails?.oid; } @@ -37,10 +38,10 @@ class FgUser { async isInGroups(grpIds) { if (!grpIds?.length) return false; - const appAt = await sharepointAuth.getAccessToken(); + const appAt = await this.sharepointAuth.getAccessToken(); // eslint-disable-next-line max-len const numGrps = grpIds.length; - let url = appConfig.getConfig().groupCheckUrl || ''; + let url = this.appConfig.getConfig().groupCheckUrl || ''; url += `&$filter=id eq '${this.userOid}'`; let found = false; for (let c = 0; c < numGrps; c += 1) { @@ -67,22 +68,22 @@ class FgUser { } async isInAdminGroup() { - const grpIds = appConfig.getConfig().fgAdminGroups; + const grpIds = this.appConfig.getConfig().fgAdminGroups; return !grpIds?.length ? false : this.isInGroups(grpIds); } async isInUserGroup() { - const grpIds = appConfig.getConfig().fgUserGroups; + const grpIds = this.appConfig.getConfig().fgUserGroups; return !grpIds?.length ? false : this.isInGroups(grpIds); } async isUser() { - const dr = await sharepoint.getDriveRoot(this.at); + const dr = await this.sharepoint.getDriveRoot(this.at); return dr ? this.isInUserGroup() : false; } async isAdmin() { - const dr = await sharepoint.getDriveRoot(this.at); + const dr = await this.sharepoint.getDriveRoot(this.at); return dr ? this.isInAdminGroup() : false; } } diff --git a/actions/helixUtils.js b/actions/helixUtils.js index b465338..ef9384b 100644 --- a/actions/helixUtils.js +++ b/actions/helixUtils.js @@ -16,7 +16,6 @@ ************************************************************************* */ const fetch = require('node-fetch'); -const appConfig = require('./appConfig'); const { getAioLogger, delay } = require('./utils'); const MAX_RETRIES = 5; @@ -30,18 +29,22 @@ const LIVE = 'live'; const logger = getAioLogger(); class HelixUtils { + constructor(appConfig) { + this.appConfig = appConfig; + } + getOperations() { return { PREVIEW, LIVE }; } getRepo(isFloodgate = false, fgColor = 'pink') { - const urlInfo = appConfig.getUrlInfo(); + const urlInfo = this.appConfig.getUrlInfo(); return isFloodgate ? `${urlInfo.getRepo()}-${fgColor}` : urlInfo.getRepo(); } getAdminApiKey(isFloodgate = false, fgColor = 'pink') { const repo = this.getRepo(isFloodgate, fgColor); - const { helixAdminApiKeys = {} } = appConfig.getConfig(); + const { helixAdminApiKeys = {} } = this.appConfig.getConfig(); return helixAdminApiKeys[repo]; } @@ -53,7 +56,7 @@ class HelixUtils { */ canBulkPreviewPublish(isFloodgate = false, fgColor = 'pink') { const repo = this.getRepo(isFloodgate, fgColor); - const { enablePreviewPublish } = appConfig.getConfig(); + const { enablePreviewPublish } = this.appConfig.getConfig(); const repoRegexArr = enablePreviewPublish.map((ps) => new RegExp(`^${ps}$`)); return true && repoRegexArr.find((rx) => rx.test(repo)); } @@ -74,7 +77,7 @@ class HelixUtils { } try { const repo = this.getRepo(isFloodgate, fgColor); - const urlInfo = appConfig.getUrlInfo(); + const urlInfo = this.appConfig.getUrlInfo(); const bulkUrl = `https://admin.hlx.page/${operation}/${urlInfo.getOwner()}/${repo}/${urlInfo.getBranch()}/*`; const options = { method: 'POST', @@ -127,29 +130,29 @@ class HelixUtils { async bulkJobStatus(jobName, operation, repo, bulkPreviewStatus = {}, retryAttempt = 1) { logger.info(`Checking job status of ${jobName} for ${operation}`); try { - const { helixAdminApiKeys } = appConfig.getConfig(); + const { helixAdminApiKeys } = this.appConfig.getConfig(); const options = {}; if (helixAdminApiKeys && helixAdminApiKeys[repo]) { options.headers = new fetch.Headers(); options.headers.append('Authorization', `token ${helixAdminApiKeys[repo]}`); } const bulkOperation = operation === LIVE ? PUBLISH : operation; - const urlInfo = appConfig.getUrlInfo(); + const urlInfo = this.appConfig.getUrlInfo(); const statusUrl = `https://admin.hlx.page/job/${urlInfo.getOwner()}/${repo}/${urlInfo.getBranch()}/${bulkOperation}/${jobName}/details`; const response = await fetch(statusUrl, options); logger.info(`Status call response ${response.ok} with status ${response.status} `); - if (!response.ok && retryAttempt <= appConfig.getConfig().maxBulkPreviewChecks) { - await delay(appConfig.getConfig().bulkPreviewCheckInterval * 1000); + if (!response.ok && retryAttempt <= this.appConfig.getConfig().maxBulkPreviewChecks) { + await delay(this.appConfig.getConfig().bulkPreviewCheckInterval * 1000); await this.bulkJobStatus(jobName, operation, repo, bulkPreviewStatus, retryAttempt + 1); } else if (response.ok) { const jobStatusJson = await response.json(); - logger.info(`${operation} progress ${JSON.stringify(jobStatusJson.progress)}`); + logger.info(`${operation} state ${JSON.stringify(jobStatusJson.state)} progress ${JSON.stringify(jobStatusJson.progress)}`); jobStatusJson.data?.resources?.forEach((rs) => { bulkPreviewStatus[rs.path] = { success: JOB_STATUS_CODES.includes(rs.status) }; }); if (jobStatusJson.state !== 'stopped' && !jobStatusJson.cancelled && - retryAttempt <= appConfig.getConfig().maxBulkPreviewChecks) { - await delay(appConfig.getConfig().bulkPreviewCheckInterval * 1000); + retryAttempt <= this.appConfig.getConfig().maxBulkPreviewChecks) { + await delay(this.appConfig.getConfig().bulkPreviewCheckInterval * 1000); await this.bulkJobStatus(jobName, operation, repo, bulkPreviewStatus, retryAttempt + 1); } } @@ -160,4 +163,4 @@ class HelixUtils { } } -module.exports = new HelixUtils(); +module.exports = HelixUtils; diff --git a/actions/maint/maint.js b/actions/maint/maint.js index 27e868e..89c396c 100644 --- a/actions/maint/maint.js +++ b/actions/maint/maint.js @@ -19,8 +19,8 @@ const openwhisk = require('openwhisk'); const filesLib = require('@adobe/aio-lib-files'); const { getAioLogger, PROMOTE_ACTION } = require('../utils'); -const appConfig = require('../appConfig'); -const sharepointAuth = require('../sharepointAuth'); +const AppConfig = require('../appConfig'); +const SharepointAuth = require('../sharepointAuth'); const FgStatus = require('../fgStatus'); const FgUser = require('../fgUser'); @@ -40,10 +40,10 @@ async function main(args) { clearStateStore: args.clearStateStore, tracker: args.tracker, }; - appConfig.setAppConfig(args); + const appConfig = new AppConfig(args); const filesSdk = await filesLib.init(); - const fgUser = new FgUser({ at: args.spToken }); - const maintAction = new MaintAction(); + const fgUser = new FgUser({ appConfig }); + const maintAction = new MaintAction(appConfig); maintAction.setFilesSdk(filesSdk); // Admin function @@ -55,11 +55,11 @@ async function main(args) { if (!(payload.permissions.isAdmin || payload.permissions.isUser)) { payload.error = 'Could not determine the user.'; logger.error(payload); - return exitAction({ + return { payload, - }); + }; } - const userDetails = sharepointAuth.getUserDetails(args.spToken); + const userDetails = new SharepointAuth(appConfig.getMsalConfig()).getUserDetails(args.spToken); logger.info(`maint action ${JSON.stringify(params)} by ${JSON.stringify(userDetails)}`); if (params.listFilePath !== undefined) payload.fileList = await maintAction.listFiles(params.listFilePath); @@ -73,15 +73,19 @@ async function main(args) { payload.error = err; } - return exitAction({ + return { payload, - }); + }; } class MaintAction { + constructor(appConfig) { + this.appConfig = appConfig; + } + setFilesSdk(filesSdk) { this.filesSdk = filesSdk; - this.filesSdkPath = appConfig.getBatchConfig().batchFilesPath; + this.filesSdkPath = this.appConfig.getBatchConfig().batchFilesPath; return this; } @@ -150,9 +154,4 @@ class MaintAction { } } -function exitAction(resp) { - appConfig.removePayload(); - return resp; -} - exports.main = main; diff --git a/actions/project.js b/actions/project.js index c7d3af0..6a53e27 100644 --- a/actions/project.js +++ b/actions/project.js @@ -15,82 +15,84 @@ * from Adobe. ************************************************************************* */ -const { getExcelTable, getFilesData } = require('./sharepoint'); const { getAioLogger, getDocPathFromUrl } = require('./utils'); const PROJECT_URL_TBL = 'URL'; -async function getProjectDetails(projectExcelPath) { - const logger = getAioLogger(); - logger.info('Getting paths from project excel worksheet'); +class Project { + constructor({ sharepoint }) { + this.sharepoint = sharepoint; + } + + async getProjectDetails(projectExcelPath) { + const logger = getAioLogger(); + logger.info('Getting paths from project excel worksheet'); - const urlsData = await getExcelTable(projectExcelPath, PROJECT_URL_TBL); - const urls = new Map(); - const filePaths = new Map(); - // UrlsData Sample [[['/one']],[['/two']],[['/two"]],[['/three']]] - const uniqueRows = urlsData - .filter((cols) => cols?.length && cols[0]) - .reduce((accumulator, currentValue) => { - const existingItem = accumulator.find((cols) => cols[0][0] === currentValue[0][0]); - if (!existingItem) { - accumulator.push(currentValue); + const urlsData = await this.sharepoint.getExcelTable(projectExcelPath, PROJECT_URL_TBL); + const urls = new Map(); + const filePaths = new Map(); + // UrlsData Sample [[['/one']],[['/two']],[['/two"]],[['/three']]] + const uniqueRows = urlsData + .filter((cols) => cols?.length && cols[0]) + .reduce((accumulator, currentValue) => { + const existingItem = accumulator.find((cols) => cols[0][0] === currentValue[0][0]); + if (!existingItem) { + accumulator.push(currentValue); + } + return accumulator; + }, []); + uniqueRows.forEach((cols) => { + const url = cols?.length && cols[0]; + const docPath = getDocPathFromUrl(url); + urls.set(url, { doc: { filePath: docPath, url } }); + // Add urls data to filePaths map + if (filePaths.has(docPath)) { + filePaths.get(docPath).push(url); + } else { + filePaths.set(docPath, [url]); } - return accumulator; - }, []); - uniqueRows.forEach((cols) => { - const url = cols?.length && cols[0]; - const docPath = getDocPathFromUrl(url); - urls.set(url, { doc: { filePath: docPath, url } }); - // Add urls data to filePaths map - if (filePaths.has(docPath)) { - filePaths.get(docPath).push(url); - } else { - filePaths.set(docPath, [url]); - } - }); + }); - return { - urls, filePaths - }; -} + return { + urls, filePaths + }; + } -/** - * Makes the sharepoint file data part of `projectDetail` per URL. - */ -function injectSharepointData(projectUrls, filePaths, docPaths, spFiles) { - for (let i = 0; i < spFiles.length; i += 1) { - let fileBody = {}; - let status = 404; - if (spFiles[i].fileSize) { - fileBody = spFiles[i]; - status = 200; + /** + * Makes the sharepoint file data part of `projectDetail` per URL. + */ + injectSharepointData(projectUrls, filePaths, docPaths, spFiles) { + for (let i = 0; i < spFiles.length; i += 1) { + let fileBody = {}; + let status = 404; + if (spFiles[i].fileSize) { + fileBody = spFiles[i]; + status = 200; + } + const filePath = docPaths[i]; + const urls = filePaths.get(filePath); + urls.forEach((key) => { + const urlObjVal = projectUrls.get(key); + urlObjVal.doc.sp = fileBody; + urlObjVal.doc.sp.status = status; + }); } - const filePath = docPaths[i]; - const urls = filePaths.get(filePath); - urls.forEach((key) => { - const urlObjVal = projectUrls.get(key); - urlObjVal.doc.sp = fileBody; - urlObjVal.doc.sp.status = status; - }); } -} -async function updateProjectWithDocs(projectDetail) { - const logger = getAioLogger(); - if (!projectDetail || !projectDetail.filePaths) { - const errorMessage = 'Error occurred when injecting sharepoint data'; - logger.error(errorMessage); - throw new Error(errorMessage); + async updateProjectWithDocs(projectDetail) { + const logger = getAioLogger(); + if (!projectDetail || !projectDetail.filePaths) { + const errorMessage = 'Error occurred when injecting sharepoint data'; + logger.error(errorMessage); + throw new Error(errorMessage); + } + const { filePaths } = projectDetail; + const docPaths = [...filePaths.keys()]; + const spFiles = await this.sharepoint.getFilesData(docPaths); + this.injectSharepointData(projectDetail.urls, filePaths, docPaths, spFiles); } - const { filePaths } = projectDetail; - const docPaths = [...filePaths.keys()]; - const spFiles = await getFilesData(docPaths); - injectSharepointData(projectDetail.urls, filePaths, docPaths, spFiles); } -module.exports = { - getProjectDetails, - updateProjectWithDocs, -}; +module.exports = Project; diff --git a/actions/promote/createBatch.js b/actions/promote/createBatch.js index 2346bee..b3b4c9e 100644 --- a/actions/promote/createBatch.js +++ b/actions/promote/createBatch.js @@ -15,20 +15,16 @@ * from Adobe. ************************************************************************* */ const openwhisk = require('openwhisk'); -const { getConfig } = require('../config'); const { - getAuthorizedRequestOption, fetchWithRetry -} = require('../sharepoint'); -const { - getAioLogger, logMemUsage, getInstanceKey, isFilePatternMatched, PROMOTE_ACTION + getAioLogger, logMemUsage, getInstanceKey, PROMOTE_ACTION } = require('../utils'); -const FgAction = require('../FgAction'); +const FgAction = require('../fgAction'); const FgStatus = require('../fgStatus'); const BatchManager = require('../batchManager'); -const appConfig = require('../appConfig'); +const AppConfig = require('../appConfig'); +const FgPromoteActionHelper = require('../fgPromoteActionHelper'); const logger = getAioLogger(); -const MAX_CHILDREN = 5000; /** * This createBatch has following functions @@ -58,19 +54,21 @@ async function main(params) { }; const ow = openwhisk(); // Initialize action - const fgAction = new FgAction(PROMOTE_ACTION, params); + const appConfig = new AppConfig(params); + const fgAction = new FgAction(PROMOTE_ACTION, appConfig); fgAction.init({ ow, skipUserDetails: true }); const { fgStatus } = fgAction.getActionParams(); const siteFgRootPath = appConfig.getSiteFgRootPath(); - const batchManager = new BatchManager({ key: PROMOTE_ACTION, instanceKey: getInstanceKey({ fgRootFolder: siteFgRootPath }) }); + const batchManager = new BatchManager({ key: PROMOTE_ACTION, instanceKey: getInstanceKey({ fgRootFolder: siteFgRootPath }), batchConfig: appConfig.getBatchConfig() }); await batchManager.init(); // For current cleanup files before starting await batchManager.cleanupFiles(); try { const vStat = await fgAction.validateAction(valParams); if (vStat && vStat.code !== 200) { - return exitAction(vStat); + return vStat; } + const fgPromoteActionHelper = new FgPromoteActionHelper(); respPayload = 'Getting all files to be promoted.'; await fgStatus.updateStatusToStateLib({ @@ -80,7 +78,7 @@ async function main(params) { logger.info(respPayload); respPayload = 'Creating batches.'; logger.info(respPayload); - respPayload = await createBatch(batchManager, appConfig); + respPayload = await fgPromoteActionHelper.createBatch(batchManager, appConfig); await fgStatus.updateStatusToStateLib({ status: FgStatus.PROJECT_STATUS.IN_PROGRESS, statusMessage: respPayload, @@ -100,72 +98,9 @@ async function main(params) { respPayload = err; } - return exitAction({ + return { body: respPayload, - }); -} - -/** - * Find all files in the FG tree to promote. Add to batches. - */ -async function createBatch(batchManager, appConf) { - const { sp } = await getConfig(); - const options = await getAuthorizedRequestOption({ method: 'GET' }); - const promoteIgnoreList = appConf.getPromoteIgnorePaths(); - logger.info(`Promote ignore list: ${promoteIgnoreList}`); - - // Temporarily restricting the iteration for promote to under /drafts folder only - return findAndBatchFGFiles({ - baseURI: sp.api.file.get.fgBaseURI, - options, - fgFolders: appConf.isDraftOnly() ? ['/drafts'] : [''], - promoteIgnoreList, - downloadBaseURI: sp.api.file.download.baseURI - }, batchManager); -} - -/** - * Iteratively finds all files under a specified root folder. Add them to batches - */ -async function findAndBatchFGFiles( - { - baseURI, options, fgFolders, promoteIgnoreList, downloadBaseURI - }, batchManager -) { - const fgRoot = baseURI.split(':').pop(); - const pPathRegExp = new RegExp(`.*:${fgRoot}`); - while (fgFolders.length !== 0) { - const uri = `${baseURI}${fgFolders.shift()}:/children?$top=${MAX_CHILDREN}`; - // eslint-disable-next-line no-await-in-loop - const res = await fetchWithRetry(uri, options); - if (res.ok) { - // eslint-disable-next-line no-await-in-loop - const json = await res.json(); - // eslint-disable-next-line no-await-in-loop - const driveItems = json.value; - for (let di = 0; di < driveItems?.length; di += 1) { - const item = driveItems[di]; - const itemPath = `${item.parentReference.path.replace(pPathRegExp, '')}/${item.name}`; - if (!isFilePatternMatched(itemPath, promoteIgnoreList)) { - if (item.folder) { - // it is a folder - fgFolders.push(itemPath); - } else { - const downloadUrl = `${downloadBaseURI}/${item.id}/content`; - // eslint-disable-next-line no-await-in-loop - await batchManager.addFile({ fileDownloadUrl: downloadUrl, filePath: itemPath }); - } - } else { - logger.info(`Ignored from promote: ${itemPath}`); - } - } - } - } -} - -function exitAction(resp) { - appConfig.removePayload(); - return resp; + }; } exports.main = main; diff --git a/actions/promote/postCopyWorker.js b/actions/promote/postCopyWorker.js index f236cf3..043f27b 100644 --- a/actions/promote/postCopyWorker.js +++ b/actions/promote/postCopyWorker.js @@ -17,13 +17,14 @@ ************************************************************************* */ const openwhisk = require('openwhisk'); const { - handleExtension, getAioLogger, logMemUsage, getInstanceKey, PROMOTE_ACTION, PROMOTE_BATCH + getAioLogger, logMemUsage, getInstanceKey, PROMOTE_ACTION, PROMOTE_BATCH } = require('../utils'); -const helixUtils = require('../helixUtils'); -const FgAction = require('../FgAction'); +const HelixUtils = require('../helixUtils'); +const FgAction = require('../fgAction'); const FgStatus = require('../fgStatus'); const BatchManager = require('../batchManager'); -const appConfig = require('../appConfig'); +const AppConfig = require('../appConfig'); +const FgPromoteActionHelper = require('../fgPromoteActionHelper'); async function main(params) { const logger = getAioLogger(); @@ -39,18 +40,19 @@ async function main(params) { const ow = openwhisk(); // Initialize action logger.info(`Post promote worker started for batch ${batchNumber}`); - const fgAction = new FgAction(`${PROMOTE_BATCH}_${batchNumber}`, params); + const appConfig = new AppConfig(params); + const fgAction = new FgAction(`${PROMOTE_BATCH}_${batchNumber}`, appConfig); fgAction.init({ ow, skipUserDetails: true, fgStatusParams: { keySuffix: `Batch_${batchNumber}` } }); const { fgStatus } = fgAction.getActionParams(); const fgRootFolder = appConfig.getSiteFgRootPath(); let respPayload; - const batchManager = new BatchManager({ key: PROMOTE_ACTION, instanceKey: getInstanceKey({ fgRootFolder }) }); + const batchManager = new BatchManager({ key: PROMOTE_ACTION, instanceKey: getInstanceKey({ fgRootFolder }), batchConfig: appConfig.getBatchConfig() }); await batchManager.init({ batchNumber }); try { const vStat = await fgAction.validateAction(valParams); if (vStat && vStat.code !== 200) { - return exitAction(vStat); + return vStat; } respPayload = 'Previewing/Publishing promoted content'; @@ -59,8 +61,9 @@ async function main(params) { status: FgStatus.PROJECT_STATUS.IN_PROGRESS, statusMessage: respPayload }); - - respPayload = await previewPublish(appConfig.getDoPublish(), batchManager); + const helixUtils = new HelixUtils(appConfig); + const fgPromoteActionHelper = new FgPromoteActionHelper(); + respPayload = await fgPromoteActionHelper.previewPublish(appConfig.getDoPublish(), { batchManager, helixUtils }); await fgStatus.updateStatusToStateLib({ status: FgStatus.PROJECT_STATUS.COMPLETED, statusMessage: respPayload @@ -74,64 +77,9 @@ async function main(params) { respPayload = err; } - return exitAction({ + return { body: respPayload, - }); -} - -async function previewPublish(doPublish, batchManager) { - const logger = getAioLogger(); - - let stepMsg = 'Getting all batch files.'; - // Get the batch files using the batchmanager for the assigned batch and process them - const currentBatch = await batchManager.getCurrentBatch(); - const currBatchLbl = `Batch-${currentBatch.getBatchNumber()}`; - const allFloodgatedFiles = await currentBatch.getFiles(); - const promotedFiles = allFloodgatedFiles.map((e) => e.file.filePath); - const resultsContent = await currentBatch.getResultsContent() || {}; - const failedPromotes = resultsContent.failedPromotes || []; - const prevPaths = promotedFiles.filter((item) => !failedPromotes.includes(item)).map((e) => handleExtension(e)); - logger.info(`Post promote files for ${currBatchLbl} are ${prevPaths?.length}`); - - logger.info('Previewing promoted files.'); - let previewStatuses = []; - let publishStatuses = []; - if (helixUtils.canBulkPreviewPublish()) { - previewStatuses = await helixUtils.bulkPreviewPublish(prevPaths, helixUtils.getOperations().PREVIEW); - stepMsg = 'Completed generating Preview for promoted files.'; - logger.info(stepMsg); - - if (doPublish) { - stepMsg = 'Publishing promoted files.'; - logger.info(stepMsg); - publishStatuses = await helixUtils.bulkPreviewPublish(prevPaths, helixUtils.getOperations().LIVE); - stepMsg = 'Completed Publishing for promoted files'; - logger.info(stepMsg); - } - } - - const failedPreviews = previewStatuses.filter((status) => !status.success) - .map((status) => status.path); - const failedPublishes = publishStatuses.filter((status) => !status.success) - .map((status) => status.path); - logger.info(`Post promote ${currBatchLbl}, Prm: ${failedPromotes?.length}, Prv: ${failedPreviews?.length}, Pub: ${failedPublishes?.length}`); - - if (failedPromotes.length > 0 || failedPreviews.length > 0 || failedPublishes.length > 0) { - stepMsg = 'Error occurred when promoting floodgated content. Check project excel sheet for additional information.'; - logger.info(stepMsg); - // Write the information to batch manifest - currentBatch.writeResults({ failedPromotes, failedPreviews, failedPublishes }); - throw new Error(stepMsg); - } - logMemUsage(); - logger.info(`All tasks for promote ${currBatchLbl} is completed`); - stepMsg = 'All tasks for floodgate promote is completed'; - return stepMsg; -} - -function exitAction(resp) { - appConfig.removePayload(); - return resp; + }; } exports.main = main; diff --git a/actions/promote/promote.js b/actions/promote/promote.js index 6291a98..90a1de0 100644 --- a/actions/promote/promote.js +++ b/actions/promote/promote.js @@ -14,15 +14,14 @@ * is strictly forbidden unless prior written permission is obtained * from Adobe. ************************************************************************* */ - // eslint-disable-next-line import/no-extraneous-dependencies const openwhisk = require('openwhisk'); const { getAioLogger, PROMOTE_ACTION } = require('../utils'); const FgStatus = require('../fgStatus'); -const FgAction = require('../FgAction'); -const appConfig = require('../appConfig'); +const FgAction = require('../fgAction'); +const AppConfig = require('../appConfig'); // This returns the activation ID of the action that it called async function main(args) { @@ -38,7 +37,8 @@ async function main(args) { }; const ow = openwhisk(); // Initialize action - const fgAction = new FgAction(PROMOTE_ACTION, args); + const appConfig = new AppConfig(args); + const fgAction = new FgAction(PROMOTE_ACTION, appConfig); fgAction.init({ ow }); const { fgStatus } = fgAction.getActionParams(); @@ -46,7 +46,7 @@ async function main(args) { // Validations const vStat = await fgAction.validateAction(valParams); if (vStat && vStat.code !== 200) { - return exitAction(vStat); + return vStat; } fgAction.logStart(); @@ -58,13 +58,13 @@ async function main(args) { }); logger.info(`FGStatus store ${await fgStatus.getStatusFromStateLib()}`); - return exitAction(ow.actions.invoke({ + return ow.actions.invoke({ name: 'milo-fg/promote-create-batch', blocking: false, // this is the flag that instructs to execute the worker asynchronous result: false, params: appConfig.getPassthruParams() }).then(async (result) => { - // attaching activation id to the status + // attaching activation id to the status respPayload = await fgStatus.updateStatusToStateLib({ status: FgStatus.PROJECT_STATUS.IN_PROGRESS, activationId: result.activationId @@ -82,7 +82,7 @@ async function main(args) { code: 500, payload: respPayload }; - })); + }); } catch (err) { logger.error(err); respPayload = await fgStatus.updateStatusToStateLib({ @@ -91,15 +91,10 @@ async function main(args) { }); } - return exitAction({ + return { code: 500, payload: respPayload, - }); -} - -function exitAction(resp) { - appConfig.removePayload(); - return resp; + }; } exports.main = main; diff --git a/actions/promote/triggerNTrack.js b/actions/promote/triggerNTrack.js index 1254192..c3f17b5 100644 --- a/actions/promote/triggerNTrack.js +++ b/actions/promote/triggerNTrack.js @@ -17,14 +17,14 @@ ************************************************************************* */ const openwhisk = require('openwhisk'); -const { updateExcelTable } = require('../sharepoint'); +const Sharepoint = require('../sharepoint'); const { toUTCStr, getAioLogger, PROMOTE_ACTION, PROMOTE_BATCH, actInProgress } = require('../utils'); -const appConfig = require('../appConfig'); +const AppConfig = require('../appConfig'); const FgStatus = require('../fgStatus'); const BatchManager = require('../batchManager'); -const FgAction = require('../FgAction'); +const FgAction = require('../fgAction'); const logger = getAioLogger(); @@ -40,19 +40,20 @@ async function main(params) { }; const ow = openwhisk(); - appConfig.setAppConfig(params); - const batchManager = new BatchManager({ key: PROMOTE_ACTION }); + let appConfig = new AppConfig(params); + const batchManager = new BatchManager({ key: PROMOTE_ACTION, batchConfig: appConfig.getBatchConfig() }); await batchManager.init(); // Read instance_info.json const instanceContent = await batchManager.getInstanceData(); if (!instanceContent || !instanceContent.dtls) { - return exitAction({ body: 'None to run!' }); + return { body: 'None to run!' }; } const { batchesInfo } = instanceContent.dtls; // Initialize action - const fgAction = new FgAction(PROMOTE_ACTION, { ...params, ...instanceContent.dtls }); + appConfig = new AppConfig({ ...params, ...instanceContent.dtls }); + const fgAction = new FgAction(PROMOTE_ACTION, appConfig); fgAction.init({ ow, skipUserDetails: true }); const { fgStatus } = fgAction.getActionParams(); const { payload } = appConfig.getConfig(); @@ -60,7 +61,7 @@ async function main(params) { try { const vStat = await fgAction.validateAction(valParams); if (vStat && vStat.code !== 200) { - return exitAction(vStat); + return vStat; } // Checks how many batches are in progress and the total batch count @@ -77,14 +78,15 @@ async function main(params) { }); // Check to see all batches are complete - const { anyInProg, allCopyDone } = await checkBatchesInProg(payload.fgRootFolder, batchesInfo, ow); + const { anyInProg, allCopyDone } = await checkBatchesInProg(appConfig, batchesInfo, ow); await batchManager.writeToInstanceFile(instanceContent); // Collect status and mark as complete + const sharepoint = new Sharepoint(appConfig); if (allCopyDone) { - const allDone = await checkPostPromoteStatus(payload.fgRootFolder, batchesInfo); + const allDone = await checkPostPromoteStatus(appConfig, batchesInfo); if (allDone) { - await completePromote(payload.projectExcelPath, batchesInfo, batchManager, fgStatus); + await completePromote(payload.projectExcelPath, batchesInfo, batchManager, fgStatus, { sharepoint }); } await batchManager.writeToInstanceFile(instanceContent); } else if (!anyInProg) { @@ -118,19 +120,19 @@ async function main(params) { logger.info('Error while updatnig failed status'); } } - return exitAction({ + return { body: respPayload, - }); + }; } /** * Checks if activation is in progress by inspecting state and activations - * @param {*} fgRootFolder Root folder + * @param {*} appConfig Payload and configs * @param {*} actDtls activation details like activation id * @param {*} ow Openwisk api interface * @returns flag if any activation is in progress */ -async function checkBatchesInProg(fgRootFolder, actDtls, ow) { +async function checkBatchesInProg(appConfig, actDtls, ow) { let fgStatus; let batchState; let batchInProg = false; @@ -145,7 +147,8 @@ async function checkBatchesInProg(fgRootFolder, actDtls, ow) { if (activationId && !copyDone) { fgStatus = new FgStatus({ action: `${PROMOTE_BATCH}_${batchNumber}`, - keySuffix: `Batch_${batchNumber}` + keySuffix: `Batch_${batchNumber}`, + appConfig }); batchState = await fgStatus.getStatusFromStateLib().then((result) => result?.action); // Track and trigger called before the state is marked in progress with activation id @@ -168,7 +171,7 @@ async function checkBatchesInProg(fgRootFolder, actDtls, ow) { return { anyInProg: batchInProg, allCopyDone }; } -async function checkPostPromoteStatus(fgRootFolder, actDtls) { +async function checkPostPromoteStatus(appConfig, actDtls) { let fgStatus; let batchState; let batchInProg = false; @@ -184,7 +187,8 @@ async function checkPostPromoteStatus(fgRootFolder, actDtls) { if (activationId && copyDone && !done) { fgStatus = new FgStatus({ action: `${PROMOTE_BATCH}_${batchNumber}`, - keySuffix: `Batch_${batchNumber}` + keySuffix: `Batch_${batchNumber}`, + appConfig }); batchState = await fgStatus.getStatusFromStateLib().then((result) => result?.action); // Track and trigger called before the state is marked in progress with activation id @@ -240,8 +244,9 @@ async function triggerPromoteWorkerAction(ow, params, fgStatus) { * @param {*} actDtls activation details like id * @param {*} batchManager BatchManager to get batch details like path * @param {*} fgStatus Floodgate status instance to update state + * @param {*} options Additional parameters like sharepoint for SP related operations */ -async function completePromote(projectExcelPath, actDtls, batchManager, fgStatus) { +async function completePromote(projectExcelPath, actDtls, batchManager, fgStatus, { sharepoint }) { let batchNumber; let results; const failedPromotes = []; @@ -283,15 +288,11 @@ async function completePromote(projectExcelPath, actDtls, batchManager, fgStatus const { startTime: startPromote, endTime: endPromote } = fgStatus.getStartEndTime(); const excelValues = [['PROMOTE', toUTCStr(startPromote), toUTCStr(endPromote), failedPromotes.join('\n'), failedPreviews.join('\n'), failedPublishes.join('\n')]]; - await updateExcelTable(projectExcelPath, 'PROMOTE_STATUS', excelValues); + await sharepoint.updateExcelTable(projectExcelPath, 'PROMOTE_STATUS', excelValues); logger.info('Project excel file updated with promote status.'); await batchManager.markComplete(fgErrors ? { failedPromotes, failedPreviews, failedPublishes } : null); logger.info('Marked complete in batch manager.'); } -function exitAction(resp) { - appConfig.removePayload(); - return resp; -} exports.main = main; diff --git a/actions/promote/worker.js b/actions/promote/worker.js index b9b3fe7..a2b286c 100644 --- a/actions/promote/worker.js +++ b/actions/promote/worker.js @@ -16,17 +16,14 @@ * from Adobe. ************************************************************************* */ const openwhisk = require('openwhisk'); -const { getConfig } = require('../config'); -const { - getAuthorizedRequestOption, saveFile, getFileUsingDownloadUrl, fetchWithRetry -} = require('../sharepoint'); const { getAioLogger, delay, logMemUsage, getInstanceKey, PROMOTE_ACTION, PROMOTE_BATCH } = require('../utils'); -const FgAction = require('../FgAction'); +const FgAction = require('../fgAction'); const FgStatus = require('../fgStatus'); const BatchManager = require('../batchManager'); -const appConfig = require('../appConfig'); +const AppConfig = require('../appConfig'); +const FgPromoteActionHelper = require('../fgPromoteActionHelper'); const DELAY_TIME_PROMOTE = 3000; @@ -44,19 +41,21 @@ async function main(params) { const ow = openwhisk(); // Initialize action logger.info(`Promote started for ${batchNumber}`); - const fgAction = new FgAction(`${PROMOTE_BATCH}_${batchNumber}`, params); + const appConfig = new AppConfig(params); + const fgAction = new FgAction(`${PROMOTE_BATCH}_${batchNumber}`, appConfig); fgAction.init({ ow, skipUserDetails: true, fgStatusParams: { keySuffix: `Batch_${batchNumber}` } }); const { fgStatus } = fgAction.getActionParams(); const fgRootFolder = appConfig.getSiteFgRootPath(); let respPayload; - const batchManager = new BatchManager({ key: PROMOTE_ACTION, instanceKey: getInstanceKey({ fgRootFolder }) }); + const batchManager = new BatchManager({ key: PROMOTE_ACTION, instanceKey: getInstanceKey({ fgRootFolder }), batchConfig: appConfig.getBatchConfig() }); await batchManager.init({ batchNumber }); try { const vStat = await fgAction.validateAction(valParams); if (vStat && vStat.code !== 200) { - return exitAction(vStat); + return vStat; } + const fgPromoteActionHelper = new FgPromoteActionHelper(); await fgStatus.clearState(); @@ -72,7 +71,7 @@ async function main(params) { }); respPayload = 'Promote files'; logger.info(respPayload); - respPayload = await promoteFloodgatedFiles(batchManager); + respPayload = await fgPromoteActionHelper.promoteFloodgatedFiles(batchManager, appConfig); respPayload = `Promoted files ${JSON.stringify(respPayload)}`; await fgStatus.updateStatusToStateLib({ status: FgStatus.PROJECT_STATUS.IN_PROGRESS, @@ -93,115 +92,9 @@ async function main(params) { respPayload = err; } - return exitAction({ + return { body: respPayload, - }); -} - -/** - * Copies the Floodgated files back to the main content tree. - * Creates intermediate folders if needed. - */ -async function promoteCopy(srcPath, destinationFolder) { - const { sp } = await getConfig(); - const { baseURI } = sp.api.file.copy; - const rootFolder = baseURI.split('/').pop(); - const payload = { ...sp.api.file.copy.payload, parentReference: { path: `${rootFolder}${destinationFolder}` } }; - const options = await getAuthorizedRequestOption({ - method: sp.api.file.copy.method, - body: JSON.stringify(payload), - }); - - // copy source is the pink directory for promote - const copyStatusInfo = await fetchWithRetry(`${sp.api.file.copy.fgBaseURI}${srcPath}:/copy?@microsoft.graph.conflictBehavior=replace`, options); - const statusUrl = copyStatusInfo.headers.get('Location'); - let copySuccess = false; - let copyStatusJson = {}; - while (statusUrl && !copySuccess && copyStatusJson.status !== 'failed') { - // eslint-disable-next-line no-await-in-loop - const status = await fetchWithRetry(statusUrl); - if (status.ok) { - // eslint-disable-next-line no-await-in-loop - copyStatusJson = await status.json(); - copySuccess = copyStatusJson.status === 'completed'; - } - } - return copySuccess; -} - -async function promoteFloodgatedFiles(batchManager) { - const logger = getAioLogger(); - - async function promoteFile(batchItem) { - const { fileDownloadUrl, filePath } = batchItem.file; - const status = { success: false, srcPath: filePath }; - try { - let promoteSuccess = false; - const destinationFolder = `${filePath.substring(0, filePath.lastIndexOf('/'))}`; - const copyFileStatus = await promoteCopy(filePath, destinationFolder); - if (copyFileStatus) { - promoteSuccess = true; - } else { - const file = await getFileUsingDownloadUrl(fileDownloadUrl); - const saveStatus = await saveFile(file, filePath); - if (saveStatus.success) { - promoteSuccess = true; - } - } - status.success = promoteSuccess; - } catch (error) { - const errorMessage = `Error promoting files ${fileDownloadUrl} at ${filePath} to main content tree ${error.message}`; - logger.error(errorMessage); - status.success = false; - } - return status; - } - - let i = 0; - let stepMsg = 'Getting all floodgated files to promote.'; - // Get the batch files using the batchmanager for the assigned batch and process them - const currentBatch = await batchManager.getCurrentBatch(); - const currBatchLbl = `Batch-${currentBatch.getBatchNumber()}`; - const allFloodgatedFiles = await currentBatch?.getFiles(); - logger.info(`Files for the batch are ${allFloodgatedFiles.length}`); - // create batches to process the data - const batchArray = []; - const numBulkReq = appConfig.getNumBulkReq(); - for (i = 0; i < allFloodgatedFiles.length; i += numBulkReq) { - const arrayChunk = allFloodgatedFiles.slice(i, i + numBulkReq); - batchArray.push(arrayChunk); - } - - // process data in batches - const promoteStatuses = []; - for (i = 0; i < batchArray.length; i += 1) { - // eslint-disable-next-line no-await-in-loop - promoteStatuses.push(...await Promise.all( - batchArray[i].map((bi) => promoteFile(bi)) - )); - // eslint-disable-next-line no-await-in-loop, no-promise-executor-return - await delay(DELAY_TIME_PROMOTE); - } - - stepMsg = `Completed promoting all documents in the batch ${currBatchLbl}`; - logger.info(stepMsg); - - const failedPromotes = promoteStatuses.filter((status) => !status.success) - .map((status) => status.srcPath || 'Path Info Not available'); - logger.info(`Promote ${currBatchLbl}, Prm: ${failedPromotes?.length}`); - - if (failedPromotes.length > 0) { - stepMsg = 'Error occurred when promoting floodgated content. Check project excel sheet for additional information.'; - logger.info(stepMsg); - // Write the information to batch manifest - await currentBatch.writeResults({ failedPromotes }); - } else { - stepMsg = `Promoted floodgate for ${currBatchLbl} successfully`; - logger.info(stepMsg); - } - logMemUsage(); - stepMsg = `Floodgate promote (copy) of ${currBatchLbl} is completed`; - return stepMsg; + }; } async function triggerPostCopy(ow, params, fgStatus) { @@ -229,9 +122,4 @@ async function triggerPostCopy(ow, params, fgStatus) { }); } -function exitAction(resp) { - appConfig.removePayload(); - return resp; -} - exports.main = main; diff --git a/actions/promoteStatus/promoteStatus.js b/actions/promoteStatus/promoteStatus.js index 747a645..0bed8c5 100644 --- a/actions/promoteStatus/promoteStatus.js +++ b/actions/promoteStatus/promoteStatus.js @@ -19,7 +19,7 @@ const { errorResponse, getAioLogger, getInstanceKey, PROMOTE_ACTION } = require('../utils'); -const appConfig = require('../appConfig'); +const AppConfig = require('../appConfig'); const FgUser = require('../fgUser'); const BatchManager = require('../batchManager'); @@ -32,9 +32,6 @@ const GEN_ERROR_SC = 500; * Returns promote status. The details of the status needed are passed a arguments. * Sample Input * { - * "promoteStatus": true, - * "batchFiles": 2, - * "batchResults": 2, * "promoteResults": true, * "fgShareUrl": "https://adobe.sharepoint.com/:f:/r/sites/adobecom/Shared%20Documents/milo-pink?web=1", * "spToken": "" @@ -69,22 +66,22 @@ const GEN_ERROR_SC = 500; async function main(args) { const payload = {}; try { - appConfig.setAppConfig(args); + const appConfig = new AppConfig(args); const batchNumber = args.batchFiles || args.batchResults; // Validations - const fgUser = new FgUser({ at: args.spToken }); + const fgUser = new FgUser({ appConfig }); if (!args.fgShareUrl) { - return exitAction(errorResponse(BAD_REQUEST_SC, 'Mising required fgShareUrl parameter')); + return errorResponse(BAD_REQUEST_SC, 'Mising required fgShareUrl parameter'); } if (!await fgUser.isUser()) { - return exitAction(errorResponse(AUTH_FAILED_SC, 'Authentication failed. Please refresh page and try again.')); + return errorResponse(AUTH_FAILED_SC, 'Authentication failed. Please refresh page and try again.'); } // Starts const siteFgRootPath = appConfig.getSiteFgRootPath(); - const batchManager = new BatchManager({ key: PROMOTE_ACTION, instanceKey: getInstanceKey({ fgRootFolder: siteFgRootPath }) }); + const batchManager = new BatchManager({ key: PROMOTE_ACTION, instanceKey: getInstanceKey({ fgRootFolder: siteFgRootPath }), batchConfig: appConfig.getBatchConfig() }); await batchManager.init({ batchNumber }); const currentBatch = batchNumber ? await batchManager.getCurrentBatch() : null; @@ -113,17 +110,12 @@ async function main(args) { } } catch (err) { logger.error(err); - return exitAction(errorResponse(GEN_ERROR_SC, `Something went wrong: ${err}`)); + return errorResponse(GEN_ERROR_SC, `Something went wrong: ${err}`); } - return exitAction({ + return { ...payload - }); -} - -function exitAction(resp) { - appConfig.removePayload(); - return resp; + }; } exports.main = main; diff --git a/actions/sharepoint.js b/actions/sharepoint.js index 21bcaf2..af05247 100644 --- a/actions/sharepoint.js +++ b/actions/sharepoint.js @@ -17,10 +17,8 @@ const { Headers } = require('node-fetch'); const fetch = require('node-fetch'); -const { getConfig } = require('./config'); const { getAioLogger } = require('./utils'); -const appConfig = require('./appConfig'); -const sharepointAuth = require('./sharepointAuth'); +const SharepointAuth = require('./sharepointAuth'); const SP_CONN_ERR_LST = ['ETIMEDOUT', 'ECONNRESET']; const APP_USER_AGENT = 'ISV|Adobe|MiloFloodgate/0.1.0'; @@ -34,443 +32,439 @@ const LOG_RESP_HEADER = false; let nextCallAfter = 0; const itemIdMap = {}; -// eslint-disable-next-line default-param-last -async function getAuthorizedRequestOption({ body = null, json = true, method = 'GET' } = {}) { - const appSpToken = await sharepointAuth.getAccessToken(); - const bearer = `Bearer ${appSpToken}`; - - const headers = new Headers(); - headers.append('Authorization', bearer); - headers.append('User-Agent', APP_USER_AGENT); - if (json) { - headers.append('Accept', 'application/json'); - headers.append('Content-Type', 'application/json'); +class Sharepoint { + constructor(appConfig) { + this.appConfig = appConfig; + this.sharepointAuth = new SharepointAuth(this.appConfig.getMsalConfig()); } - const options = { - method, - headers, - }; - - if (body) { - options.body = typeof body === 'string' ? body : JSON.stringify(body); - } - - return options; -} - -async function executeGQL(url, opts) { - const options = await getAuthorizedRequestOption(opts); - const res = await fetchWithRetry(url, options); - if (!res.ok) { - throw new Error(`Failed to execute ${url}`); + getSharepointAuth() { + return this.sharepointAuth; } - return res.json(); -} -async function getItemId(uri, path) { - const key = `~${uri}~${path}~`; - itemIdMap[key] = itemIdMap[key] || await executeGQL(`${uri}${path}?$select=id`); - return itemIdMap[key]?.id; -} + // eslint-disable-next-line default-param-last + async getAuthorizedRequestOption({ body = null, json = true, method = 'GET' } = {}) { + const appSpToken = await this.sharepointAuth.getAccessToken(); + const bearer = `Bearer ${appSpToken}`; -async function getDriveRoot(accessToken) { - const logger = getAioLogger(); - try { const headers = new Headers(); - headers.append('Authorization', `Bearer ${accessToken}`); + headers.append('Authorization', bearer); headers.append('User-Agent', APP_USER_AGENT); - headers.append('Accept', 'application/json'); - const fgSite = appConfig.getFgSite(); - const response = await fetchWithRetry(`${fgSite}/drive/root`, { headers }); + if (json) { + headers.append('Accept', 'application/json'); + headers.append('Content-Type', 'application/json'); + } + + const options = { + method, + headers, + }; - if (response?.ok) { - const driveDtls = await response.json(); - return driveDtls; + if (body) { + options.body = typeof body === 'string' ? body : JSON.stringify(body); } - logger.info(`Unable to get User details: ${response?.status}`); - } catch (error) { - logger.info('Unable to fetch User Info'); - logger.info(JSON.stringify(error)); - } - return null; -} -async function getFileData(filePath, isFloodgate) { - const { sp } = await getConfig(); - const options = await getAuthorizedRequestOption(); - const baseURI = isFloodgate ? sp.api.directory.create.fgBaseURI : sp.api.directory.create.baseURI; - const resp = await fetchWithRetry(`${baseURI}${filePath}`, options); - const json = await resp.json(); - const fileDownloadUrl = json['@microsoft.graph.downloadUrl']; - const fileSize = json.size; - return { fileDownloadUrl, fileSize }; -} + return options; + } -async function getFilesData(filePaths, isFloodgate) { - const batchArray = []; - for (let i = 0; i < filePaths.length; i += BATCH_REQUEST_LIMIT) { - const arrayChunk = filePaths.slice(i, i + BATCH_REQUEST_LIMIT); - batchArray.push(arrayChunk); + async executeGQL(url, opts) { + const options = await this.getAuthorizedRequestOption(opts); + const res = await this.fetchWithRetry(url, options); + if (!res.ok) { + throw new Error(`Failed to execute ${url}`); + } + return res.json(); } - // process data in batches - const fileJsonResp = []; - for (let i = 0; i < batchArray.length; i += 1) { - // eslint-disable-next-line no-await-in-loop - fileJsonResp.push(...await Promise.all( - batchArray[i].map((file) => getFileData(file, isFloodgate)), - )); - // eslint-disable-next-line no-await-in-loop, no-promise-executor-return - await new Promise((resolve) => setTimeout(resolve, BATCH_DELAY_TIME)); + + async getItemId(uri, path) { + const key = `~${uri}~${path}~`; + itemIdMap[key] = itemIdMap[key] || await this.executeGQL(`${uri}${path}?$select=id`); + return itemIdMap[key]?.id; } - return fileJsonResp; -} -async function getFile(doc) { - if (doc && doc.sp && doc.sp.status === 200) { - const response = await fetchWithRetry(doc.sp.fileDownloadUrl); - return response.blob(); + async getDriveRoot(accessToken) { + const logger = getAioLogger(); + try { + const headers = new Headers(); + headers.append('Authorization', `Bearer ${accessToken}`); + headers.append('User-Agent', APP_USER_AGENT); + headers.append('Accept', 'application/json'); + const fgSite = this.appConfig.getFgSite(); + const response = await this.fetchWithRetry(`${fgSite}/drive/root`, { headers }); + + if (response?.ok) { + const driveDtls = await response.json(); + return driveDtls; + } + logger.info(`Unable to get User details: ${response?.status}`); + } catch (error) { + logger.info('Unable to fetch User Info'); + logger.info(JSON.stringify(error)); + } + return null; } - return undefined; -} -async function getFileUsingDownloadUrl(downloadUrl) { - const options = await getAuthorizedRequestOption({ json: false }); - const response = await fetchWithRetry(downloadUrl, options); - if (response) { - return response.blob(); + async getFileData(filePath, isFloodgate) { + const sp = await this.appConfig.getSpConfig(); + const options = await this.getAuthorizedRequestOption(); + const baseURI = isFloodgate ? sp.api.directory.create.fgBaseURI : sp.api.directory.create.baseURI; + const resp = await this.fetchWithRetry(`${baseURI}${filePath}`, options); + const json = await resp.json(); + const fileDownloadUrl = json['@microsoft.graph.downloadUrl']; + const fileSize = json.size; + return { fileDownloadUrl, fileSize }; } - return undefined; -} -async function createFolder(folder, isFloodgate) { - const { sp } = await getConfig(); - const options = await getAuthorizedRequestOption({ method: sp.api.directory.create.method }); - options.body = JSON.stringify(sp.api.directory.create.payload); + async getFilesData(filePaths, isFloodgate) { + const batchArray = []; + for (let i = 0; i < filePaths.length; i += BATCH_REQUEST_LIMIT) { + const arrayChunk = filePaths.slice(i, i + BATCH_REQUEST_LIMIT); + batchArray.push(arrayChunk); + } + // process data in batches + const fileJsonResp = []; + for (let i = 0; i < batchArray.length; i += 1) { + // eslint-disable-next-line no-await-in-loop + fileJsonResp.push(...await Promise.all( + batchArray[i].map((file) => this.getFileData(file, isFloodgate)), + )); + // eslint-disable-next-line no-await-in-loop, no-promise-executor-return + await new Promise((resolve) => setTimeout(resolve, BATCH_DELAY_TIME)); + } + return fileJsonResp; + } - const baseURI = isFloodgate ? sp.api.directory.create.fgBaseURI : sp.api.directory.create.baseURI; - const res = await fetchWithRetry(`${baseURI}${folder}`, options); - if (res.ok) { - return res.json(); + async getFile(doc) { + if (doc && doc.sp && doc.sp.status === 200) { + const response = await this.fetchWithRetry(doc.sp.fileDownloadUrl); + return response.blob(); + } + return undefined; } - throw new Error(`Could not create folder: ${folder}`); -} -function getFolderFromPath(path) { - if (path.includes('.')) { - return path.substring(0, path.lastIndexOf('/')); + async getFileUsingDownloadUrl(downloadUrl) { + const options = await this.getAuthorizedRequestOption({ json: false }); + const response = await this.fetchWithRetry(downloadUrl, options); + if (response) { + return response.blob(); + } + return undefined; } - return path; -} -function getFileNameFromPath(path) { - return path.split('/').pop().split('/').pop(); -} + async createFolder(folder, isFloodgate) { + const sp = await this.appConfig.getSpConfig(); + const options = await this.getAuthorizedRequestOption({ method: sp.api.directory.create.method }); + options.body = JSON.stringify(sp.api.directory.create.payload); -async function createUploadSession(sp, file, dest, filename, isFloodgate) { - const payload = { - ...sp.api.file.createUploadSession.payload, - description: 'Preview file', - fileSize: file.size, - name: filename, - }; - const options = await getAuthorizedRequestOption({ method: sp.api.file.createUploadSession.method }); - options.body = JSON.stringify(payload); + const baseURI = isFloodgate ? sp.api.directory.create.fgBaseURI : sp.api.directory.create.baseURI; + const res = await this.fetchWithRetry(`${baseURI}${folder}`, options); + if (res.ok) { + return res.json(); + } + throw new Error(`Could not create folder: ${folder}`); + } - const baseURI = isFloodgate ? sp.api.file.createUploadSession.fgBaseURI : sp.api.file.createUploadSession.baseURI; + getFolderFromPath(path) { + if (path.includes('.')) { + return path.substring(0, path.lastIndexOf('/')); + } + return path; + } - const createdUploadSession = await fetchWithRetry(`${baseURI}${dest}:/createUploadSession`, options); - return createdUploadSession.ok ? createdUploadSession.json() : undefined; -} + getFileNameFromPath(path) { + return path.split('/').pop().split('/').pop(); + } -async function uploadFile(sp, uploadUrl, file) { - const options = await getAuthorizedRequestOption({ - json: false, - method: sp.api.file.upload.method, - }); - // TODO API is limited to 60Mb, for more, we need to batch the upload. - options.headers.append('Content-Length', file.size); - options.headers.append('Content-Range', `bytes 0-${file.size - 1}/${file.size}`); - options.headers.append('Prefer', 'bypass-shared-lock'); - options.body = file; - return fetchWithRetry(`${uploadUrl}`, options); -} + async createUploadSession(sp, file, dest, filename, isFloodgate) { + const payload = { + ...sp.api.file.createUploadSession.payload, + description: 'Preview file', + fileSize: file.size, + name: filename, + }; + const options = await this.getAuthorizedRequestOption({ method: sp.api.file.createUploadSession.method }); + options.body = JSON.stringify(payload); -async function deleteFile(sp, filePath) { - const options = await getAuthorizedRequestOption({ - json: false, - method: sp.api.file.delete.method, - }); - options.headers.append('Prefer', 'bypass-shared-lock'); - return fetch(filePath, options); -} + const baseURI = isFloodgate ? sp.api.file.createUploadSession.fgBaseURI : sp.api.file.createUploadSession.baseURI; -async function renameFile(spFileUrl, filename) { - const options = await getAuthorizedRequestOption({ method: 'PATCH', body: JSON.stringify({ name: filename }) }); - options.headers.append('Prefer', 'bypass-shared-lock'); - return fetch(spFileUrl, options); -} + const createdUploadSession = await this.fetchWithRetry(`${baseURI}${dest}:/createUploadSession`, options); + return createdUploadSession.ok ? createdUploadSession.json() : undefined; + } -async function releaseUploadSession(sp, uploadUrl) { - await deleteFile(sp, uploadUrl); -} + async uploadFile(sp, uploadUrl, file) { + const options = await this.getAuthorizedRequestOption({ + json: false, + method: sp.api.file.upload.method, + }); + // TODO API is limited to 60Mb, for more, we need to batch the upload. + options.headers.append('Content-Length', file.size); + options.headers.append('Content-Range', `bytes 0-${file.size - 1}/${file.size}`); + options.headers.append('Prefer', 'bypass-shared-lock'); + options.body = file; + return this.fetchWithRetry(`${uploadUrl}`, options); + } -function getLockedFileNewName(filename) { - const extIndex = filename.indexOf('.'); - const fileNameWithoutExtn = filename.substring(0, extIndex); - const fileExtn = filename.substring(extIndex); - return `${fileNameWithoutExtn}-locked-${Date.now()}${fileExtn}`; -} + async deleteFile(sp, filePath) { + const options = await this.getAuthorizedRequestOption({ + json: false, + method: sp.api.file.delete.method, + }); + options.headers.append('Prefer', 'bypass-shared-lock'); + return fetch(filePath, options); + } -async function createSessionAndUploadFile(sp, file, dest, filename, isFloodgate) { - const createdUploadSession = await createUploadSession(sp, file, dest, filename, isFloodgate); - const status = {}; - if (createdUploadSession) { - const uploadSessionUrl = createdUploadSession.uploadUrl; - if (!uploadSessionUrl) { - return status; - } - status.sessionUrl = uploadSessionUrl; - const uploadedFile = await uploadFile(sp, uploadSessionUrl, file); - if (!uploadedFile) { - return status; - } - if (uploadedFile.ok) { - status.uploadedFile = await uploadedFile.json(); - status.success = true; - } else if (uploadedFile.status === 423) { - status.locked = true; - } + async renameFile(spFileUrl, filename) { + const options = await this.getAuthorizedRequestOption({ method: 'PATCH', body: JSON.stringify({ name: filename }) }); + options.headers.append('Prefer', 'bypass-shared-lock'); + return fetch(spFileUrl, options); } - return status; -} -/** - * The method gets the list of files, extracts the parent path, extracts uniq paths, - * filters common parents urls - * e.g.. [/a/b/one.txt, /a/b/two.txt, /a/c/three.txt, /a/c/d/three.txt] - * Folders to create would be [/a/b, /a/c/d] - * This triggers async and waits for batch to complete. These are small batches so should be fast. - * The $batch can be used in future to submit only one URL - * @param {*} srcPathList Paths of files for which folder creating is needed - * @param {*} isFloodgate Is floodgate flag - * @returns Create folder status - */ -async function bulkCreateFolders(srcPathList, isFloodgate) { - const logger = getAioLogger(); - const createtFolderStatuses = []; - const allPaths = srcPathList.map((e) => { - if (e.length < 2 || !e[1]?.doc) return ''; - return getFolderFromPath(e[1].doc.filePath); - }).filter((e) => true && e); - const uniqPathLst = Array.from(new Set(allPaths)); - const leafPathLst = uniqPathLst.filter((e) => uniqPathLst.findIndex((e1) => e1.indexOf(`${e}/`) >= 0) < 0); - // logger.info(`Unique path list ${JSON.stringify(leafPathLst)}`); - try { - logger.info('bulkCreateFolders started'); - const promises = leafPathLst.map((folder) => createFolder(folder, isFloodgate)); - logger.info('Got createfolder promises and waiting....'); - createtFolderStatuses.push(...await Promise.all(promises)); - logger.info(`bulkCreateFolders completed ${createtFolderStatuses?.length}`); - // logger.info(`bulkCreateFolders statuses ${JSON.stringify(createtFolderStatuses)}`); - } catch (error) { - logger.info('Error while creating folders'); - logger.info(error?.stack); + async releaseUploadSession(sp, uploadUrl) { + await this.deleteFile(sp, uploadUrl); } - logger.info(`bulkCreateFolders returning ${createtFolderStatuses?.length}`); - return createtFolderStatuses; -} -async function copyFile(srcPath, destinationFolder, newName, isFloodgate, isFloodgateLockedFile) { - const logger = getAioLogger(); - const { sp } = await getConfig(); - const { baseURI, fgBaseURI } = sp.api.file.copy; - const rootFolder = isFloodgate ? fgBaseURI.split('/').pop() : baseURI.split('/').pop(); + getLockedFileNewName(filename) { + const extIndex = filename.indexOf('.'); + const fileNameWithoutExtn = filename.substring(0, extIndex); + const fileExtn = filename.substring(extIndex); + return `${fileNameWithoutExtn}-locked-${Date.now()}${fileExtn}`; + } - const payload = { ...sp.api.file.copy.payload, parentReference: { path: `${rootFolder}${destinationFolder}` } }; - if (newName) { - payload.name = newName; + async createSessionAndUploadFile(sp, file, dest, filename, isFloodgate) { + const createdUploadSession = await this.createUploadSession(sp, file, dest, filename, isFloodgate); + const status = {}; + if (createdUploadSession) { + const uploadSessionUrl = createdUploadSession.uploadUrl; + if (!uploadSessionUrl) { + return status; + } + status.sessionUrl = uploadSessionUrl; + const uploadedFile = await this.uploadFile(sp, uploadSessionUrl, file); + if (!uploadedFile) { + return status; + } + if (uploadedFile.ok) { + status.uploadedFile = await uploadedFile.json(); + status.success = true; + } else if (uploadedFile.status === 423) { + status.locked = true; + } + } + return status; } - const options = await getAuthorizedRequestOption({ - method: sp.api.file.copy.method, - body: JSON.stringify(payload), - }); - // In case of FG copy action triggered via saveFile(), locked file copy happens in the floodgate content location - // So baseURI is updated to reflect the destination accordingly - const contentURI = isFloodgate && isFloodgateLockedFile ? fgBaseURI : baseURI; - const copyStatusInfo = await fetchWithRetry(`${contentURI}${srcPath}:/copy?@microsoft.graph.conflictBehavior=replace`, options); - const statusUrl = copyStatusInfo.headers.get('Location'); - let copySuccess = false; - let copyStatusJson = {}; - if (!statusUrl) { - logger.info(`Copy of ${srcPath} returned ${copyStatusInfo?.status} with no followup URL`); + + /** + * The method gets the list of files, extracts the parent path, extracts uniq paths, + * filters common parents urls + * e.g.. [/a/b/one.txt, /a/b/two.txt, /a/c/three.txt, /a/c/d/three.txt] + * Folders to create would be [/a/b, /a/c/d] + * This triggers async and waits for batch to complete. These are small batches so should be fast. + * The $batch can be used in future to submit only one URL + * @param {*} srcPathList Paths of files for which folder creating is needed + * @param {*} isFloodgate Is floodgate flag + * @returns Create folder status + */ + async bulkCreateFolders(srcPathList, isFloodgate) { + const logger = getAioLogger(); + const createtFolderStatuses = []; + const allPaths = srcPathList.map((e) => { + if (e.length < 2 || !e[1]?.doc) return ''; + return this.getFolderFromPath(e[1].doc.filePath); + }).filter((e) => true && e); + const uniqPathLst = Array.from(new Set(allPaths)); + const leafPathLst = uniqPathLst.filter((e) => uniqPathLst.findIndex((e1) => e1.indexOf(`${e}/`) >= 0) < 0); + // logger.info(`Unique path list ${JSON.stringify(leafPathLst)}`); + try { + logger.info('bulkCreateFolders started'); + const promises = leafPathLst.map((folder) => this.createFolder(folder, isFloodgate)); + logger.info('Got createfolder promises and waiting....'); + createtFolderStatuses.push(...await Promise.all(promises)); + logger.info(`bulkCreateFolders completed ${createtFolderStatuses?.length}`); + // logger.info(`bulkCreateFolders statuses ${JSON.stringify(createtFolderStatuses)}`); + } catch (error) { + logger.info('Error while creating folders'); + logger.info(error?.stack); + } + logger.info(`bulkCreateFolders returning ${createtFolderStatuses?.length}`); + return createtFolderStatuses; } - while (statusUrl && !copySuccess && copyStatusJson.status !== 'failed') { - // eslint-disable-next-line no-await-in-loop - const status = await fetchWithRetry(statusUrl); - if (status.ok) { + + async copyFile(srcPath, destinationFolder, newName, isFloodgate, isFloodgateLockedFile) { + const logger = getAioLogger(); + const sp = await this.appConfig.getSpConfig(); + const { baseURI, fgBaseURI } = sp.api.file.copy; + const rootFolder = isFloodgate ? fgBaseURI.split('/').pop() : baseURI.split('/').pop(); + + const payload = { ...sp.api.file.copy.payload, parentReference: { path: `${rootFolder}${destinationFolder}` } }; + if (newName) { + payload.name = newName; + } + const options = await this.getAuthorizedRequestOption({ + method: sp.api.file.copy.method, + body: JSON.stringify(payload), + }); + // In case of FG copy action triggered via saveFile(), locked file copy happens in the floodgate content location + // So baseURI is updated to reflect the destination accordingly + const contentURI = isFloodgate && isFloodgateLockedFile ? fgBaseURI : baseURI; + const copyStatusInfo = await this.fetchWithRetry(`${contentURI}${srcPath}:/copy?@microsoft.graph.conflictBehavior=replace`, options); + const statusUrl = copyStatusInfo.headers.get('Location'); + let copySuccess = false; + let copyStatusJson = {}; + if (!statusUrl) { + logger.info(`Copy of ${srcPath} returned ${copyStatusInfo?.status} with no followup URL`); + } + while (statusUrl && !copySuccess && copyStatusJson.status !== 'failed') { // eslint-disable-next-line no-await-in-loop - copyStatusJson = await status.json(); - copySuccess = copyStatusJson.status === 'completed'; + const status = await this.fetchWithRetry(statusUrl); + if (status.ok) { + // eslint-disable-next-line no-await-in-loop + copyStatusJson = await status.json(); + copySuccess = copyStatusJson.status === 'completed'; + } } + return copySuccess; } - return copySuccess; -} -async function saveFile(file, dest, isFloodgate) { - try { - const folder = getFolderFromPath(dest); - const filename = getFileNameFromPath(dest); - await createFolder(folder, isFloodgate); - const { sp } = await getConfig(); - let uploadFileStatus = await createSessionAndUploadFile(sp, file, dest, filename, isFloodgate); - if (uploadFileStatus.locked) { - await releaseUploadSession(sp, uploadFileStatus.sessionUrl); - const lockedFileNewName = getLockedFileNewName(filename); - const baseURI = isFloodgate ? sp.api.file.get.fgBaseURI : sp.api.file.get.baseURI; - const spFileUrl = `${baseURI}${dest}`; - await renameFile(spFileUrl, lockedFileNewName); - const newLockedFilePath = `${folder}/${lockedFileNewName}`; - const copyFileStatus = await copyFile(newLockedFilePath, folder, filename, isFloodgate, true); - if (copyFileStatus) { - uploadFileStatus = await createSessionAndUploadFile(sp, file, dest, filename, isFloodgate); - if (uploadFileStatus.success) { - await deleteFile(sp, `${baseURI}${newLockedFilePath}`); + async saveFile(file, dest, isFloodgate) { + try { + const folder = this.getFolderFromPath(dest); + const filename = this.getFileNameFromPath(dest); + await this.createFolder(folder, isFloodgate); + const sp = await this.appConfig.getSpConfig(); + let uploadFileStatus = await this.createSessionAndUploadFile(sp, file, dest, filename, isFloodgate); + if (uploadFileStatus.locked) { + await this.releaseUploadSession(sp, uploadFileStatus.sessionUrl); + const lockedFileNewName = this.getLockedFileNewName(filename); + const baseURI = isFloodgate ? sp.api.file.get.fgBaseURI : sp.api.file.get.baseURI; + const spFileUrl = `${baseURI}${dest}`; + await this.renameFile(spFileUrl, lockedFileNewName); + const newLockedFilePath = `${folder}/${lockedFileNewName}`; + const copyFileStatus = await this.copyFile(newLockedFilePath, folder, filename, isFloodgate, true); + if (copyFileStatus) { + uploadFileStatus = await this.createSessionAndUploadFile(sp, file, dest, filename, isFloodgate); + if (uploadFileStatus.success) { + await this.deleteFile(sp, `${baseURI}${newLockedFilePath}`); + } } } + const uploadedFileJson = uploadFileStatus.uploadedFile; + if (uploadedFileJson) { + return { success: true, uploadedFileJson, path: dest }; + } + } catch (error) { + return { success: false, path: dest, errorMsg: error.message }; } - const uploadedFileJson = uploadFileStatus.uploadedFile; - if (uploadedFileJson) { - return { success: true, uploadedFileJson, path: dest }; + return { success: false, path: dest }; + } + + async getExcelTable(excelPath, tableName) { + const sp = await this.appConfig.getSpConfig(); + const itemId = await this.getItemId(sp.api.file.get.baseURI, excelPath); + if (itemId) { + const tableJson = await this.executeGQL(`${sp.api.excel.get.baseItemsURI}/${itemId}/workbook/tables/${tableName}/rows`); + return !tableJson?.value ? [] : + tableJson.value + .filter((e) => e.values?.find((rw) => rw.find((col) => col))) + .map((e) => e.values); } - } catch (error) { - return { success: false, path: dest, errorMsg: error.message }; + return []; } - return { success: false, path: dest }; -} -async function getExcelTable(excelPath, tableName) { - const { sp } = await getConfig(); - const itemId = await getItemId(sp.api.file.get.baseURI, excelPath); - if (itemId) { - const tableJson = await executeGQL(`${sp.api.excel.get.baseItemsURI}/${itemId}/workbook/tables/${tableName}/rows`); - return !tableJson?.value ? [] : - tableJson.value - .filter((e) => e.values?.find((rw) => rw.find((col) => col))) - .map((e) => e.values); + async deleteFloodgateDir() { + const logger = getAioLogger(); + logger.info('Deleting content started.'); + const sp = await this.appConfig.getSpConfig(); + let deleteSuccess = false; + + const { fgDirPattern } = this.appConfig.getConfig(); + const fgRegExp = new RegExp(fgDirPattern); + logger.info(fgRegExp); + if (fgRegExp.test(sp.api.file.update.fgBaseURI)) { + const temp = '/temp'; + const finalBaserURI = `${sp.api.file.delete.fgBaseURI}${temp}`; + logger.info(`Deleting the folder ${finalBaserURI} `); + try { + await this.deleteFile(sp, finalBaserURI); + deleteSuccess = true; + } catch (error) { + logger.info(`Error occurred when trying to delete files of main content tree ${error.message}`); + } + } + return deleteSuccess; } - return []; -} -async function deleteFloodgateDir() { - const logger = getAioLogger(); - logger.info('Deleting content started.'); - const { sp } = await getConfig(); - let deleteSuccess = false; - - const { fgDirPattern } = appConfig.getConfig(); - const fgRegExp = new RegExp(fgDirPattern); - logger.info(fgRegExp); - if (fgRegExp.test(sp.api.file.update.fgBaseURI)) { - const temp = '/temp'; - const finalBaserURI = `${sp.api.file.delete.fgBaseURI}${temp}`; - logger.info(`Deleting the folder ${finalBaserURI} `); - try { - await deleteFile(sp, finalBaserURI); - deleteSuccess = true; - } catch (error) { - logger.info(`Error occurred when trying to delete files of main content tree ${error.message}`); + async updateExcelTable(excelPath, tableName, values) { + const sp = await this.appConfig.getSpConfig(); + const itemId = await this.getItemId(sp.api.file.get.baseURI, excelPath); + if (itemId) { + return this.executeGQL(`${sp.api.excel.update.baseItemsURI}/${itemId}/workbook/tables/${tableName}/rows`, { + body: JSON.stringify({ values }), + method: sp.api.excel.update.method, + }); } + return {}; } - return deleteSuccess; -} -async function updateExcelTable(excelPath, tableName, values) { - const { sp } = await getConfig(); - const itemId = await getItemId(sp.api.file.get.baseURI, excelPath); - if (itemId) { - return executeGQL(`${sp.api.excel.update.baseItemsURI}/${itemId}/workbook/tables/${tableName}/rows`, { - body: JSON.stringify({ values }), - method: sp.api.excel.update.method, + // fetch-with-retry added to check for Sharepoint RateLimit headers and 429 errors and to handle them accordingly. + async fetchWithRetry(apiUrl, options, retryCounts) { + let retryCount = retryCounts || 0; + const logger = getAioLogger(); + return new Promise((resolve, reject) => { + const currentTime = Date.now(); + if (retryCount > NUM_REQ_THRESHOLD) { + reject(); + } else if (nextCallAfter !== 0 && currentTime < nextCallAfter) { + setTimeout(() => this.fetchWithRetry(apiUrl, options, retryCount) + .then((newResp) => resolve(newResp)) + .catch((err) => reject(err)), nextCallAfter - currentTime); + } else { + retryCount += 1; + fetch(apiUrl, options).then((resp) => { + this.logHeaders(resp); + const retryAfter = resp.headers.get('ratelimit-reset') || resp.headers.get('retry-after') || 0; + if ((resp.headers.get('test-retry-status') === TOO_MANY_REQUESTS) || (resp.status === TOO_MANY_REQUESTS)) { + nextCallAfter = Date.now() + retryAfter * 1000; + logger.info(`Retry ${nextCallAfter}`); + this.fetchWithRetry(apiUrl, options, retryCount) + .then((newResp) => resolve(newResp)) + .catch((err) => reject(err)); + } else { + nextCallAfter = retryAfter ? Math.max(Date.now() + retryAfter * 1000, nextCallAfter) : nextCallAfter; + resolve(resp); + } + }).catch((err) => { + logger.warn(`Connection error ${apiUrl} with ${JSON.stringify(err)}`); + if (err && SP_CONN_ERR_LST.includes(err.code) && retryCount < NUM_REQ_THRESHOLD) { + logger.info(`Retry ${SP_CONN_ERR_LST}`); + nextCallAfter = Date.now() + RETRY_ON_CF * 1000; + return this.fetchWithRetry(apiUrl, options, retryCount) + .then((newResp) => resolve(newResp)) + .catch((err2) => reject(err2)); + } + return reject(err); + }); + } }); } - return {}; -} -// fetch-with-retry added to check for Sharepoint RateLimit headers and 429 errors and to handle them accordingly. -async function fetchWithRetry(apiUrl, options, retryCounts) { - let retryCount = retryCounts || 0; - const logger = getAioLogger(); - return new Promise((resolve, reject) => { - const currentTime = Date.now(); - if (retryCount > NUM_REQ_THRESHOLD) { - reject(); - } else if (nextCallAfter !== 0 && currentTime < nextCallAfter) { - setTimeout(() => fetchWithRetry(apiUrl, options, retryCount) - .then((newResp) => resolve(newResp)) - .catch((err) => reject(err)), nextCallAfter - currentTime); - } else { - retryCount += 1; - fetch(apiUrl, options).then((resp) => { - logHeaders(resp); - const retryAfter = resp.headers.get('ratelimit-reset') || resp.headers.get('retry-after') || 0; - if ((resp.headers.get('test-retry-status') === TOO_MANY_REQUESTS) || (resp.status === TOO_MANY_REQUESTS)) { - nextCallAfter = Date.now() + retryAfter * 1000; - logger.info(`Retry ${nextCallAfter}`); - fetchWithRetry(apiUrl, options, retryCount) - .then((newResp) => resolve(newResp)) - .catch((err) => reject(err)); - } else { - nextCallAfter = retryAfter ? Math.max(Date.now() + retryAfter * 1000, nextCallAfter) : nextCallAfter; - resolve(resp); - } - }).catch((err) => { - logger.warn(`Connection error ${apiUrl} with ${JSON.stringify(err)}`); - if (err && SP_CONN_ERR_LST.includes(err.code) && retryCount < NUM_REQ_THRESHOLD) { - logger.info(`Retry ${SP_CONN_ERR_LST}`); - nextCallAfter = Date.now() + RETRY_ON_CF * 1000; - return fetchWithRetry(apiUrl, options, retryCount) - .then((newResp) => resolve(newResp)) - .catch((err2) => reject(err2)); - } - return reject(err); - }); - } - }); -} + getHeadersStr(response) { + const headers = {}; + response?.headers?.forEach((value, name) => { + headers[name] = value; + }); + return JSON.stringify(headers); + } -function getHeadersStr(response) { - const headers = {}; - response?.headers?.forEach((value, name) => { - headers[name] = value; - }); - return JSON.stringify(headers); -} + getLogRespHeader = () => LOG_RESP_HEADER; -function logHeaders(response) { - if (!LOG_RESP_HEADER) return; - const logger = getAioLogger(); - const hdrStr = getHeadersStr(response); - const logStr = `Status is ${response.status} with headers ${hdrStr}`; + logHeaders(response) { + if (!this.getLogRespHeader()) return; + const logger = getAioLogger(); + const hdrStr = this.getHeadersStr(response); + const logStr = `Status is ${response.status} with headers ${hdrStr}`; - if (logStr.toUpperCase().indexOf('RATE') > 0 || logStr.toUpperCase().indexOf('RETRY') > 0) logger.info(logStr); + if (logStr.toUpperCase().indexOf('RATE') > 0 || logStr.toUpperCase().indexOf('RETRY') > 0) logger.info(logStr); + } } -module.exports = { - getAuthorizedRequestOption, - executeGQL, - getDriveRoot, - getExcelTable, - getFilesData, - getFile, - getFileUsingDownloadUrl, - copyFile, - saveFile, - createFolder, - updateExcelTable, - fetchWithRetry, - getFolderFromPath, - getFileNameFromPath, - bulkCreateFolders, - deleteFloodgateDir, -}; +module.exports = Sharepoint; diff --git a/actions/sharepointAuth.js b/actions/sharepointAuth.js index 1f858e7..81bc85d 100644 --- a/actions/sharepointAuth.js +++ b/actions/sharepointAuth.js @@ -17,7 +17,6 @@ const msal = require('@azure/msal-node'); const { getAioLogger } = require('./utils'); -const appConfig = require('./appConfig'); /** * Creates a new SharePoint object, that has two methods: @@ -30,20 +29,25 @@ const appConfig = require('./appConfig'); * @returns {object} Sharepoint object */ class SharepointAuth { - init() { - const msalConfig = appConfig.getMsalConfig(); + msalConfig = null; + + constructor(msalConfig) { + this.msalConfig = msalConfig; + this.init(); + } + init() { const missingConfigs = []; - if (!msalConfig.clientId) { + if (!this.msalConfig.clientId) { missingConfigs.push('CLIENT_ID'); } - if (!msalConfig.tenantId) { + if (!this.msalConfig.tenantId) { missingConfigs.push('TENANT_ID'); } - if (!msalConfig.certThumbprint) { + if (!this.msalConfig.certThumbprint) { missingConfigs.push('CERT_THUMB_PRINT'); } - if (!msalConfig.pvtKey) { + if (!this.msalConfig.pvtKey) { missingConfigs.push('PRIVATE_KEY'); } if (missingConfigs.length > 0) { @@ -55,12 +59,12 @@ class SharepointAuth { } this.authConfig = { auth: { - clientId: msalConfig.clientId, - authority: `https://login.microsoftonline.com/${msalConfig.tenantId}`, + clientId: this.msalConfig.clientId, + authority: `https://login.microsoftonline.com/${this.msalConfig.tenantId}`, knownAuthorities: ['login.microsoftonline.com'], clientCertificate: { - privateKey: msalConfig.pvtKey, - thumbprint: msalConfig.certThumbprint, + privateKey: this.msalConfig.pvtKey, + thumbprint: this.msalConfig.certThumbprint, }, }, }; @@ -122,4 +126,4 @@ class SharepointAuth { } } -module.exports = new SharepointAuth(); +module.exports = SharepointAuth; diff --git a/actions/status/status.js b/actions/status/status.js index c9e294b..38feaea 100644 --- a/actions/status/status.js +++ b/actions/status/status.js @@ -16,7 +16,7 @@ ************************************************************************* */ // eslint-disable-next-line import/no-extraneous-dependencies -const appConfig = require('../appConfig'); +const AppConfig = require('../appConfig'); const { getAioLogger, COPY_ACTION, PROMOTE_ACTION, DELETE_ACTION } = require('../utils'); @@ -33,7 +33,7 @@ async function main(args) { const logger = getAioLogger(); let payload; try { - appConfig.setAppConfig(args); + const appConfig = new AppConfig(args); const { type, shareUrl, fgShareUrl } = args; @@ -41,7 +41,7 @@ async function main(args) { payload = 'Status : Required data is not available to get the status.'; logger.error(payload); } else { - const fgStatus = new FgStatus({ action: actionMap[type] }); + const fgStatus = new FgStatus({ action: actionMap[type], appConfig }); payload = await fgStatus.getStatusFromStateLib(); } } catch (err) { @@ -49,14 +49,9 @@ async function main(args) { payload = err; } - return exitAction({ + return { payload, - }); -} - -function exitAction(resp) { - appConfig.removePayload(); - return resp; + }; } exports.main = main; diff --git a/actions/utils.js b/actions/utils.js index 3f8bd93..1caa4ca 100644 --- a/actions/utils.js +++ b/actions/utils.js @@ -67,7 +67,7 @@ function getDocPathFromUrl(url) { path = path.slice(0, -5); return `${path}.xlsx`; } - if (path.endsWith('.svg') || path.endsWith('.pdf') || path.endsWith('.mp4')) { + if (path.endsWith('.svg') || path.endsWith('.pdf')) { return path; } if (path.endsWith('/')) { diff --git a/package-lock.json b/package-lock.json index 282c861..04bd1fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@adobe/exc-app": "^0.2.17", "@azure/msal-node": "^1.17.2", "cloudevents": "^4.0.2", + "fetch-mock-jest": "^1.5.1", "node-fetch": "^2.6.0", "openwhisk": "^3.21.7", "regenerator-runtime": "^0.13.5", @@ -5633,6 +5634,29 @@ } } }, + "node_modules/fetch-mock-jest": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/fetch-mock-jest/-/fetch-mock-jest-1.5.1.tgz", + "integrity": "sha512-+utwzP8C+Pax1GSka3nFXILWMY3Er2L+s090FOgqVNrNCPp0fDqgXnAHAJf12PLHi0z4PhcTaZNTz8e7K3fjqQ==", + "dependencies": { + "fetch-mock": "^9.11.0" + }, + "engines": { + "node": ">=8.0.0" + }, + "funding": { + "type": "charity", + "url": "https://www.justgiving.com/refugee-support-europe" + }, + "peerDependencies": { + "node-fetch": "*" + }, + "peerDependenciesMeta": { + "node-fetch": { + "optional": true + } + } + }, "node_modules/fetch-retry": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-3.2.3.tgz", @@ -15809,6 +15833,14 @@ "whatwg-url": "^6.5.0" } }, + "fetch-mock-jest": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/fetch-mock-jest/-/fetch-mock-jest-1.5.1.tgz", + "integrity": "sha512-+utwzP8C+Pax1GSka3nFXILWMY3Er2L+s090FOgqVNrNCPp0fDqgXnAHAJf12PLHi0z4PhcTaZNTz8e7K3fjqQ==", + "requires": { + "fetch-mock": "^9.11.0" + } + }, "fetch-retry": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-3.2.3.tgz", diff --git a/package.json b/package.json index 9b4510e..58b04cb 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "@adobe/exc-app": "^0.2.17", "@azure/msal-node": "^1.17.2", "cloudevents": "^4.0.2", + "fetch-mock-jest": "^1.5.1", "node-fetch": "^2.6.0", "openwhisk": "^3.21.7", "regenerator-runtime": "^0.13.5", diff --git a/test/appConfig.test.js b/test/appConfig.test.js index 89463e6..f6c37f2 100644 --- a/test/appConfig.test.js +++ b/test/appConfig.test.js @@ -15,10 +15,7 @@ * from Adobe. ************************************************************************* */ -const appConfig = require('../actions/appConfig'); - -process.env.__OW_ACTIVATION_ID = 'actid'; -appConfig.initPayload(); +const AppConfig = require('../actions/appConfig'); // Mock the 'crypto' module jest.mock('crypto', () => ({ @@ -29,7 +26,9 @@ jest.mock('crypto', () => ({ let params = { spToken: 'eyJ0eXAi', - adminPageUri: 'http://localhost:3000/tools/floodgate/index.html?project=milo--adobecom&referrer=', + adminPageUri: 'https://floodgateui--milo--adobecom.hlx.page/tools/floodgate?ref=floodgateui&repo=milo&owner=adobecom&host=milo.adobe.com&project=Milo' + + '&referrer=https%3A%2F%2Fadobe.sharepoint.com%2F%3Ax%3A%2Fr%2Fsites%2Fadobecom%2F_layouts%2F15%2FDoc.aspx' + + '%3Fsourcedoc%3D%257B442C005E-8094-4EB8-A78F-48BF427A04ED%257D%26file%3DBook5.xlsx%26action%3Ddefault%26mobileredirect%3Dtrue', projectExcelPath: '/drafts/floodgate/projects/raga/fgtest1.xlsx', shareUrl: 'https://site.sharepoint.com/:f:/r/sites/adobecom/Shared%20Documents/milo?web=1', fgShareUrl: 'https://site.sharepoint.com/:f:/r/sites/adobecom/Shared%20Documents/milo-pink?web=1', @@ -66,11 +65,10 @@ params = { maxBulkPreviewChecks: '100', enablePreviewPublish: 'true' }; - +const appConfig = new AppConfig(); describe('appConfig', () => { test('set parameters', () => { appConfig.setAppConfig(params); - expect(appConfig.getPayloadKey()).toBe('actid'); expect(appConfig.getPayload()).toBeDefined(); expect(appConfig.getConfig()).toBeDefined(); expect(appConfig.getPassthruParams()).toBeDefined(); @@ -91,30 +89,36 @@ describe('appConfig', () => { expect(appConfig.getSiteFgRootPath()).toBe('/adobecom/Shared%20Documents/milo-pink'); expect(appConfig.getUrlInfo()).toMatchObject({ urlInfoMap: { - branch: 'main', origin: 'https://main--milo--adobecom.hlx.page', owner: 'adobecom', repo: 'milo', sp: '' + branch: 'floodgateui', + origin: 'https://floodgateui--milo--adobecom.hlx.page', + owner: 'adobecom', + repo: 'milo', + sp: 'https://adobe.sharepoint.com/:x:/r/sites/adobecom/_layouts/15/Doc.aspx?sourcedoc=%7B442C005E-8094-4EB8-A78F-48BF427A04ED%7D' + + '&file=Book5.xlsx&action=default&mobileredirect=true', } }); expect(appConfig.isDraftOnly()).toBeTruthy(); expect(appConfig.getDoPublish()).not.toBeTruthy(); expect(!!appConfig.getEnablePromote()).toBeFalsy(); expect(!!appConfig.getEnableDelete()).toBeFalsy(); - - appConfig.removePayload(); - expect(() => appConfig.getPayload()).toThrow(); }); test('isDraftOnly would be true when not passed', () => { const { draftsOnly, ...remParams } = params; appConfig.setAppConfig(remParams); expect(appConfig.isDraftOnly()).toBeTruthy(); - appConfig.removePayload(); }); test('isDraftOnly is false when parameter is passed', () => { const { draftsOnly, ...remParams } = params; appConfig.setAppConfig({ draftsOnly: 'false', ...remParams }); expect(appConfig.isDraftOnly()).toBeFalsy(); - appConfig.removePayload(); + }); + + test('Test sharepoint config is populated', async () => { + appConfig.setAppConfig(params); + const sp = await appConfig.getSpConfig(); + expect(sp).toMatchObject({}); }); test('Test enable delete action flags', () => { diff --git a/test/batch.test.js b/test/batch.test.js new file mode 100644 index 0000000..425e1ff --- /dev/null +++ b/test/batch.test.js @@ -0,0 +1,126 @@ +/* ************************************************************************ +* ADOBE CONFIDENTIAL +* ___________________ +* +* Copyright 2023 Adobe +* All Rights Reserved. +* +* NOTICE: All information contained herein is, and remains +* the property of Adobe and its suppliers, if any. The intellectual +* and technical concepts contained herein are proprietary to Adobe +* and its suppliers and are protected by all applicable intellectual +* property laws, including trade secret and copyright laws. +* Dissemination of this information or reproduction of this material +* is strictly forbidden unless prior written permission is obtained +* from Adobe. +************************************************************************* */ + +const { getAioLogger } = require('../actions/utils'); +const Batch = require('../actions/batch'); + +jest.mock('../actions/utils', () => ({ + getAioLogger: jest.fn(() => ({ + info: jest.fn(), + })), +})); + +const mockFilesSdk = { + write: jest.fn(), + read: jest.fn().mockResolvedValue('{}'), + list: jest.fn().mockResolvedValue([]), +}; + +describe('Batch Class Tests', () => { + let batch; + + beforeEach(() => { + batch = new Batch({ + filesSdk: mockFilesSdk, + instancePath: '/path/to/instance', + batchNumber: 1, + maxFilesPerBatch: 200, + }); + }); + + test('Batch constructor initializes properties correctly', () => { + expect(batch.params).toEqual({ + filesSdk: mockFilesSdk, + instancePath: '/path/to/instance', + batchNumber: 1, + maxFilesPerBatch: 200, + }); + expect(batch.filesSdk).toBe(mockFilesSdk); + expect(batch.instancePath).toBe('/path/to/instance'); + expect(batch.batchNumber).toBe(1); + expect(batch.maxFilesPerBatch).toBe(200); + expect(batch.batchPath).toBe('/path/to/instance/batch_1'); + expect(batch.batchInfoFile).toBe('/path/to/instance/batch_1/batch_info.json'); + expect(batch.resultsFile).toBe('/path/to/instance/batch_1/results.json'); + }); + + test('getBatchNumber returns the correct batch number', () => { + const result = batch.getBatchNumber(); + expect(result).toBe(1); + }); + + test('getBatchPath returns the correct batch path', () => { + const result = batch.getBatchPath(); + expect(result).toBe('/path/to/instance/batch_1'); + }); + + test('canAddFile returns true when files can be added', () => { + const result = batch.canAddFile(); + expect(result).toBe(true); + }); + + test('canAddFile returns false when files cannot be added', () => { + batch.batchFiles = new Array(200); // Max files reached + const result = batch.canAddFile(); + expect(result).toBe(false); + }); + + test('addFile adds file metadata to batchFiles', async () => { + const file = { name: 'file1.txt' }; + await batch.addFile(file); + expect(batch.batchFiles).toHaveLength(1); + expect(batch.batchFiles[0]).toEqual({ file, batchNumber: 1 }); + }); + + test('savePendingFiles writes batchFiles to file', async () => { + batch.batchFiles = [{ file: { name: 'file1.txt' }, batchNumber: 1 }]; + await batch.savePendingFiles(); + expect(mockFilesSdk.write).toHaveBeenCalledWith( + '/path/to/instance/batch_1/batch_info.json', + '[{"file":{"name":"file1.txt"},"batchNumber":1}]' + ); + expect(batch.batchFiles).toHaveLength(1); // batchFiles should be cleared after saving + }); + + test('savePendingFiles does nothing if batchFiles is empty', async () => { + await batch.savePendingFiles(); + }); + + test('getFiles reads batch info file and returns file contents', async () => { + mockFilesSdk.read.mockResolvedValue('{"file":{"name":"file1.txt"},"batchNumber":1}'); + const result = await batch.getFiles(); + expect(mockFilesSdk.read).toHaveBeenCalledWith('/path/to/instance/batch_1/batch_info.json'); + expect(result).toEqual({ file: { name: 'file1.txt' }, batchNumber: 1 }); + }); + + test('writeResults writes data to results file', async () => { + const data = { status: 'success' }; + await batch.writeResults(data); + expect(mockFilesSdk.write).toHaveBeenCalledWith( + '/path/to/instance/batch_1/results.json', + '{"status":"success"}' + ); + }); + + test('getResultsContent reads results file and returns parsed data', async () => { + mockFilesSdk.list.mockResolvedValue(['results.json']); + mockFilesSdk.read.mockResolvedValue('{"status":"success"}'); + const result = await batch.getResultsContent(); + expect(mockFilesSdk.read).toHaveBeenCalledWith('/path/to/instance/batch_1/results.json'); + expect(result).toEqual({ status: 'success' }); + }); +}); diff --git a/test/batchManager.test.js b/test/batchManager.test.js new file mode 100644 index 0000000..a72812c --- /dev/null +++ b/test/batchManager.test.js @@ -0,0 +1,300 @@ +/* ************************************************************************ + * ADOBE CONFIDENTIAL + * ___________________ + * + * Copyright 2023 Adobe + * All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of Adobe and its suppliers, if any. The intellectual + * and technical concepts contained herein are proprietary to Adobe + * and its suppliers and are protected by all applicable intellectual + * property laws, including trade secret and copyright laws. + * Dissemination of this information or reproduction of this material + * is strictly forbidden unless prior written permission is obtained + * from Adobe. + ************************************************************************* */ + +describe('BatchManager', () => { + Batch = null; + BatchManager = null; + params = { + key: 'promoteAction', + instanceKey: 'milo-pink', + batchConfig: { maxFilesPerBatch: 10, batchFilesPath: '/floodgate' }, + }; + + beforeAll(() => { + jest.mock('../actions/utils', () => ({ + getAioLogger: () => ({ + info: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + }), + })); + + jest.mock('@adobe/aio-lib-files', () => ({ + init: jest.fn().mockResolvedValue({ + read: jest.fn(), + write: jest.fn(), + }), + })); + + Batch = require('../actions/batch'); + BatchManager = require('../actions/batchManager'); + }); + + afterAll(() => { + jest.clearAllMocks(); + }); + + // BatchManager can be initialized with or without params + it('should initialize BatchManager without params', () => { + const batchManager = new BatchManager(params); + expect(batchManager.params).toEqual(params); + expect(batchManager.batches).toEqual([]); + expect(batchManager.batchFilesPath).toEqual('/floodgate'); + expect(batchManager.key).toEqual('promoteAction'); + expect(batchManager.bmPath).toEqual('/floodgate/promoteAction'); + expect(batchManager.bmTracker).toEqual( + '/floodgate/promoteAction/tracker.json' + ); + expect(batchManager.instanceKey).toEqual('milo-pink'); + expect(batchManager.instancePath).toEqual( + '/floodgate/promoteAction/instancemilo-pink' + ); + expect(batchManager.instanceFile).toEqual( + '/floodgate/promoteAction/instancemilo-pink/instance_info.json' + ); + expect(batchManager.resultsFile).toEqual( + '/floodgate/promoteAction/instancemilo-pink/instance_results.json' + ); + }); + + // BatchManager can initialize a batch with a batchNumber + it('should initialize a batch with a batchNumber', () => { + const newParams = { ...params, batchNumber: 1 }; + const batchManager = new BatchManager(params); + batchManager.initBatch(newParams); + expect(batchManager.currentBatchNumber).toBe(1); + expect(batchManager.currentBatch).toBeInstanceOf(Batch); + expect(batchManager.batches).toHaveLength(1); + }); + + // BatchManager can read the bmTracker file + it('should read the bmTracker file', async () => { + const filesSdkMock = { + read: jest + .fn() + .mockResolvedValue( + Buffer.from(JSON.stringify({ instanceKeys: ['key1', 'key2'] })) + ), + }; + const batchManager = new BatchManager(params); + batchManager.filesSdk = filesSdkMock; + const result = await batchManager.readBmTracker(); + expect(result).toEqual({ instanceKeys: ['key1', 'key2'] }); + expect(filesSdkMock.read).toHaveBeenCalledWith( + '/floodgate/promoteAction/tracker.json' + ); + }); + + // BatchManager can handle errors while reading bmTracker file + it('should handle errors while reading bmTracker file', async () => { + const filesSdkMock = { + read: jest.fn().mockRejectedValue(new Error('Read error')), + }; + const batchManager = new BatchManager(params); + batchManager.filesSdk = filesSdkMock; + const result = await batchManager.readBmTracker(); + expect(result).toEqual({}); + }); + + // BatchManager can handle errors while writing to bmTracker file + it('should write to bmTracker file', async () => { + const filesSdkMock = { + read: jest + .fn() + .mockResolvedValue( + Buffer.from(JSON.stringify({ instanceKeys: ['key1', 'key2'] })) + ), + write: jest.fn(), + }; + const batchManager = new BatchManager(params); + batchManager.filesSdk = filesSdkMock; + await batchManager.writeToBmTracker({ data: 'test' }); + expect(filesSdkMock.write).toHaveBeenCalledWith( + '/floodgate/promoteAction/tracker.json', + JSON.stringify({ + instanceKeys: ['key1', 'key2', 'milo-pink'], + data: 'test', + }) + ); + }); + + // BatchManager can handle errors while writing to instance file + it('should write to instance file', async () => { + const filesSdkMock = { + write: jest.fn(), + }; + const batchManager = new BatchManager(params); + batchManager.filesSdk = filesSdkMock; + await batchManager.writeToInstanceFile({ data: 'test' }); + expect(filesSdkMock.write).toHaveBeenCalledWith( + '/floodgate/promoteAction/instancemilo-pink/instance_info.json', + JSON.stringify({ data: 'test' }) + ); + }); + + it('Add to instance file', async () => { + const fileData = + '{"lastBatch":1,"dtls":{"0":{"batchNunber":1,"activationId":"a"},"data":"test"}}'; + const writtenData = + '{"lastBatch":1,"dtls":{"0":{"batchNunber":1,"activationId":"a"},"data":"test"}}'; + const filesSdkMock = { + read: () => fileData, + write: jest.fn(), + }; + const batchManager = new BatchManager(params); + batchManager.filesSdk = filesSdkMock; + await batchManager.addToInstanceFile({ data: 'test' }); + expect(filesSdkMock.write).toHaveBeenCalledWith( + '/floodgate/promoteAction/instancemilo-pink/instance_info.json', + writtenData + ); + }); + + it('should return instance data when instance key is found in tracker.json and proceed is true', async () => { + const batchManager = new BatchManager(params); + batchManager.init({}); + const readBmTrackerMock = jest + .spyOn(batchManager, 'readBmTracker') + .mockResolvedValue({ + instanceKeys: ['_milo_pink'], + _milo_pink: { done: false, proceed: true }, + }); + const initInstanceMock = jest + .spyOn(batchManager, 'initInstance') + .mockReturnValue(batchManager); + const getInstanceFileContentMock = jest + .spyOn(batchManager, 'getInstanceFileContent') + .mockResolvedValue({ data: 'instance data' }); + + const instanceData = await batchManager.getInstanceData(); + + expect(readBmTrackerMock).toHaveBeenCalled(); + expect(initInstanceMock).toHaveBeenCalledWith({ + instanceKey: '_milo_pink', + }); + expect(getInstanceFileContentMock).toHaveBeenCalled(); + expect(instanceData).toEqual({ data: 'instance data' }); + }); + + it('should save pending files in the batch when currentBatch is not null', async () => { + // Arrange + const batchManager = new BatchManager(params); + await batchManager.init({}); + const currentBatchMock = { + savePendingFiles: jest.fn(), + }; + batchManager.currentBatch = currentBatchMock; + + // Act + await batchManager.finalizeInstance(); + + // Assert + expect(currentBatchMock.savePendingFiles).toHaveBeenCalled(); + }); + it('should mark the instance as complete', async () => { + // Arrange + const batchManager = new BatchManager(params); + await batchManager.init({}); + const writeToBmTrackerMock = jest.spyOn(batchManager, 'writeToBmTracker'); + + // Act + await batchManager.markComplete(); + + // Assert + expect(writeToBmTrackerMock).toHaveBeenCalledWith({ + [`${batchManager.instanceKey}`]: { + done: true, + proceed: false, + }, + }); + }); + it('should return parsed JSON data when files are present in the results file', async () => { + const batchManager = new BatchManager(params); + await batchManager.init(params); + const mockFileProps = [{ name: 'results.json' }]; + const mockData = { file1: 'data1', file2: 'data2' }; + const mockBuffer = Buffer.from(JSON.stringify(mockData)); + batchManager.filesSdk.list = jest.fn().mockResolvedValue(mockFileProps); + batchManager.filesSdk.read = jest.fn().mockResolvedValue(mockBuffer); + const results = await batchManager.getResultsContent(); + expect(results).toEqual(mockData); + }); + + it('should delete all files in the current action instance path', async () => { + // Arrange + const batchManager = new BatchManager(params); + const filesSdkMock = { + delete: jest.fn().mockResolvedValue(), + }; + batchManager.filesSdk = filesSdkMock; + + // Act + await batchManager.cleanupFiles(); + + // Assert + expect(filesSdkMock.delete).toHaveBeenCalledWith( + '/floodgate/promoteAction/instancemilo-pink/' + ); + }); + + it('should return the current batch if it exists', async () => { + // Arrange + const newParams = { ...params, batchNumber: 2 }; + const batchManager = new BatchManager(newParams); + await batchManager.init(newParams); + const currentBatch = await batchManager.getCurrentBatch(); + + // Assert + expect(currentBatch.getBatchNumber()).toBe(2); + }); + + // Adds file metadata to current batch if it can still add files + it('should add file metadata to current batch if it can still add files', async () => { + // Arrange + const newParams = { ...params, batchNumber: 1 }; + const batchManager = new BatchManager(params); + await batchManager.init(newParams); + const file = 'path/to/file.txt'; + jest.spyOn(Batch.prototype, 'canAddFile').mockResolvedValue(true); + const addFileMock = jest.spyOn(BatchManager.prototype, 'addFile'); + const currentBatch = batchManager.getCurrentBatch(); + + // Act + await batchManager.addFile(file); + + // Assert + expect(addFileMock).toHaveBeenCalledWith(file); + }); + + // Creates a new batch and adds file metadata if current batch is full and retry count is 0 + it('should create a new batch and add file metadata if current batch is full and retry count is 0', async () => { + // Arrange + const batchManager = new BatchManager(params); + await batchManager.init({}); + const file = 'path/to/file.txt'; + jest.spyOn(Batch.prototype, 'canAddFile').mockResolvedValue(true); + const createBatchMock = jest.spyOn(BatchManager.prototype, 'createBatch'); + const addFileMock = jest.spyOn(BatchManager.prototype, 'addFile'); + + // Act + await batchManager.addFile(file, 0); + + // Assert + expect(createBatchMock).toHaveBeenCalled(); + expect(addFileMock).toHaveBeenCalledWith(file, 1); + }); +}); diff --git a/test/fgAction.test.js b/test/fgAction.test.js new file mode 100644 index 0000000..537f795 --- /dev/null +++ b/test/fgAction.test.js @@ -0,0 +1,117 @@ +/* ************************************************************************ +* ADOBE CONFIDENTIAL +* ___________________ +* +* Copyright 2023 Adobe +* All Rights Reserved. +* +* NOTICE: All information contained herein is, and remains +* the property of Adobe and its suppliers, if any. The intellectual +* and technical concepts contained herein are proprietary to Adobe +* and its suppliers and are protected by all applicable intellectual +* property laws, including trade secret and copyright laws. +* Dissemination of this information or reproduction of this material +* is strictly forbidden unless prior written permission is obtained +* from Adobe. +************************************************************************* */ + +const AppConfig = require('../actions/appConfig'); + +// Mock the 'crypto' module +jest.mock('crypto', () => ({ + createPrivateKey: jest.fn().mockReturnValue({ + export: jest.fn().mockReturnValue('mocked') + }) +})); + +let params = { + spToken: 'eyJ0eXAi', + adminPageUri: 'http://localhost:3000/tools/floodgate/index.html?project=milo--adobecom&referrer=', + projectExcelPath: '/drafts/floodgate/projects/raga/fgtest1.xlsx', + shareUrl: 'https://site.sharepoint.com/:f:/r/sites/adobecom/Shared%20Documents/milo?web=1', + fgShareUrl: 'https://site.sharepoint.com/:f:/r/sites/adobecom/Shared%20Documents/milo-pink?web=1', + rootFolder: '/milo', + fgRootFolder: '/milo-pink', + promoteIgnorePaths: '/gnav.docx,/.milo', + doPublish: 'false', + driveId: 'drive', + fgColor: 'purple', + draftsOnly: 'true' +}; +// Add config +params = { + ...params, + fgSite: 'https://graph.microsoft.com/v1.0/sites/site.sharepoint.com,d21', + fgClientId: '008626ae-1', + fgAuthority: 'https://login.microsoftonline.com/fa7b1b5a-', + clientId: '008626ae-1', + tenantId: 'a', + certPassword: 'a', + certKey: 'a', + certThumbprint: 'a', + skipInProgressCheck: 'true', + batchFilesPath: 'milo-floodgate/batching', + maxFilesPerBatch: '200', + numBulkReq: '20', + groupCheckUrl: 'https://graph.microsoft.com/v1.0/groups/{groupOid}/members?$count=true', + fgUserGroups: '["1"]', + fgAdminGroups: '["2"]', + fgDirPattern: '.*/sites(/.*)<', + siteRootPathRex: '(pink)$', + helixAdminApiKeys: '{"milo": "ey:","milo-pink": "dy"}', + bulkPreviewCheckInterval: '8', + maxBulkPreviewChecks: '100', + enablePreviewPublish: 'true' +}; + +describe('appConfig', () => { + test('set parameters', () => { + const appConfig = new AppConfig(params); + expect(appConfig.getPayload()).toBeDefined(); + expect(appConfig.getConfig()).toBeDefined(); + expect(appConfig.getPassthruParams()).toBeDefined(); + expect(appConfig.getPassthruParams().spToken).not.toBeDefined(); + appConfig.extractPrivateKey(); + const { + clientId, tenantId, certPassword, pvtKey = 'mocked', certThumbprint + } = params; + expect({ ...appConfig.getMsalConfig() }).toMatchObject({ + clientId, tenantId, certPassword, pvtKey, certThumbprint + }); + expect(appConfig.getFgSite()).toBe(params.fgSite); + expect(appConfig.getPromoteIgnorePaths().length).toBe(6); + expect(appConfig.getSkipInProgressCheck()).toBeTruthy(); + expect({ ...appConfig.getBatchConfig() }).toMatchObject({ batchFilesPath: params.batchFilesPath, maxFilesPerBatch: 200 }); + expect(appConfig.getNumBulkReq()).toBe(20); + expect(appConfig.extractSiteRootPath()).toBe('/'); + expect(appConfig.getSiteFgRootPath()).toBe('/adobecom/Shared%20Documents/milo-pink'); + expect(appConfig.getUrlInfo()).toMatchObject({ + urlInfoMap: { + branch: 'main', origin: 'https://main--milo--adobecom.hlx.page', owner: 'adobecom', repo: 'milo', sp: '' + } + }); + expect(appConfig.isDraftOnly()).toBeTruthy(); + expect(appConfig.getDoPublish()).not.toBeTruthy(); + expect(!!appConfig.getPdoverride()).toBeFalsy(); + expect(!!appConfig.getEdgeWorkerEndDate()).toBeFalsy(); + }); + + test('isDraftOnly would be true when not passed', () => { + const { draftsOnly, ...remParams } = params; + const appConfig = new AppConfig(params); + expect(appConfig.isDraftOnly()).toBeTruthy(); + expect(remParams).toBeDefined(); + }); + + test('isDraftOnly is false when parameter is passed', () => { + const { draftsOnly, ...remParams } = params; + const appConfig = new AppConfig({ draftsOnly: null, ...remParams }); + expect(appConfig.isDraftOnly()).toBeFalsy(); + }); + + test('Test pdoverride and edgeWorkerEndDate', () => { + const appConfig = new AppConfig({ ...params, pdoverride: 'false', edgeWorkerEndDate: 'Wed, 20 Dec 2023 13:56:49 GMT' }); + expect(appConfig.getPdoverride()).toBeFalsy(); + expect(appConfig.getEdgeWorkerEndDate().getTime()).toBe(1703080609000); + }); +}); diff --git a/test/fgStatus.test.js b/test/fgStatus.test.js new file mode 100644 index 0000000..9b919d5 --- /dev/null +++ b/test/fgStatus.test.js @@ -0,0 +1,125 @@ +/* ************************************************************************ + * ADOBE CONFIDENTIAL + * ___________________ + * + * Copyright 2023 Adobe + * All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of Adobe and its suppliers, if any. The intellectual + * and technical concepts contained herein are proprietary to Adobe + * and its suppliers and are protected by all applicable intellectual + * property laws, including trade secret and copyright laws. + * Dissemination of this information or reproduction of this material + * is strictly forbidden unless prior written permission is obtained + * from Adobe. + ************************************************************************* */ +/* eslint-disable global-require */ +jest.mock('crypto', () => ({ + createHash: () => ({ + update: () => ({ + digest: () => 'md5digest' + }) + }) +})); + +jest.mock('@adobe/aio-lib-state', () => ({ + init: jest.fn().mockResolvedValue({ + get: jest.fn().mockResolvedValue({ + value: { } + }), + put: jest.fn(), + delete: jest.fn(), + }) +})); + +const stateLib = require('@adobe/aio-lib-state'); + +describe('fgStatus', () => { + let FgStatus; + let fgStatus; + let appConfigMock; + + beforeAll(() => { + jest.mock('../actions/utils', () => ({ + getAioLogger: () => ({ + info: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + }), + })); + + appConfigMock = { + getSiteFgRootPath: jest.fn().mockReturnValue('/milo-pink'), + getPayload: jest.fn().mockReturnValue({ + projectExcelPath: '/mydoc/drafts/fg/prj1.xlsx' + }), + }; + + FgStatus = require('../actions/fgStatus'); + fgStatus = new FgStatus({ + action: 'promoteAction', + statusKey: 'skey', + keySuffix: 'suf', + appConfig: appConfigMock, + userDetails: { oid: 'oid1' }, + }); + }); + + afterAll(() => { + jest.clearAllMocks(); + }); + + it('generated store key', () => { + expect(fgStatus.getStoreKey()).toBe('skey'); + const fgStatus2 = new FgStatus({ + action: 'promoteAction', + appConfig: appConfigMock, + }); + const storeKey = fgStatus2.generateStoreKey('suf'); + expect(storeKey).toBe('/milo-pinksuf'); + }); + + it('key updated', async () => { + await fgStatus.updateStatusToStateLib({ + status: FgStatus.PROJECT_STATUS.STARTED, + statusMessage: 'Started', + activationId: 'AID', + action: {}, + startTime: new Date(), + batches: [], + details: { + v: '1' + } + }); + }); + + it('status on action finished', async () => { + (await stateLib.init()).get.mockResolvedValueOnce({ + value: { + status: FgStatus.PROJECT_STATUS.IN_PROGRESS, + statusMessage: 'Steps 2', + activationId: 'AID2', + action: {}, + } + }); + await fgStatus.updateStatusToStateLib({ + status: FgStatus.PROJECT_STATUS.COMPLETED, + statusMessage: 'Finished', + activationId: 'AID', + action: 'promote', + startTime: new Date(), + batches: [], + details: { + v: '1' + } + }); + }); + + it('clear status with publish', async () => { + const deleteSpy = jest.spyOn(await stateLib.init(), 'delete'); + await fgStatus.clearState(false); + await fgStatus.clearState(true); + expect(deleteSpy).toHaveBeenCalled(); + }); +}); diff --git a/test/fgUser.test.js b/test/fgUser.test.js new file mode 100644 index 0000000..9414f16 --- /dev/null +++ b/test/fgUser.test.js @@ -0,0 +1,91 @@ +/* ************************************************************************ + * ADOBE CONFIDENTIAL + * ___________________ + * + * Copyright 2023 Adobe + * All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of Adobe and its suppliers, if any. The intellectual + * and technical concepts contained herein are proprietary to Adobe + * and its suppliers and are protected by all applicable intellectual + * property laws, including trade secret and copyright laws. + * Dissemination of this information or reproduction of this material + * is strictly forbidden unless prior written permission is obtained + * from Adobe. + ************************************************************************* */ +/* eslint-disable global-require */ +jest.mock('node-fetch', () => require('fetch-mock-jest').sandbox()); +const fetchMock = require('node-fetch'); + +describe('fgUser', () => { + let FgUser; + let fgUser; + let appConfigMock; + + beforeAll(() => { + jest.mock('../actions/utils', () => ({ + getAioLogger: () => ({ + info: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + }), + })); + + jest.mock('../actions/sharepoint', () => (jest.fn().mockReturnValue({ + getSharepointAuth: jest.fn().mockReturnValue({ + getUserDetails: jest.fn().mockReturnValue({ + oid: 'oid1' + }), + getAccessToken: jest.fn().mockReturnValue('at') + }), + getDriveRoot: jest.fn().mockReturnValue('at') + }))); + + appConfigMock = { + getConfig: jest.fn().mockReturnValue({ + fgAdminGroups: ['a'], + fgUserGroups: ['b'], + }), + }; + + FgUser = require('../actions/fgUser'); + fgUser = new FgUser({ at: 'at', appConfig: appConfigMock }); + }); + + afterAll(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + fetchMock.reset(); + }); + + it('is an admin', async () => { + fetchMock.get('*', () => ({ + value: ['a'] + })); + const found = await fgUser.isAdmin(); + expect(found).toBe(true); + }); + + it('admin group is not defined', async () => { + appConfigMock.getConfig.mockReturnValueOnce({ fgAdminGroups: [] }); + const found = await fgUser.isAdmin(); + expect(found).toBe(false); + }); + + it('is an fg user', async () => { + fetchMock.get('*', () => ({ + value: ['b'] + })); + const found = await fgUser.isUser(); + expect(found).toBe(true); + }); + + it('fg user group is not defined', async () => { + appConfigMock.getConfig.mockReturnValueOnce({ fgUserGroups: [] }); + const found = await fgUser.isUser(); + expect(found).toBe(false); + }); +}); diff --git a/test/helixUtils.test.js b/test/helixUtils.test.js new file mode 100644 index 0000000..10e24d3 --- /dev/null +++ b/test/helixUtils.test.js @@ -0,0 +1,109 @@ +/* ************************************************************************ + * ADOBE CONFIDENTIAL + * ___________________ + * + * Copyright 2023 Adobe + * All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of Adobe and its suppliers, if any. The intellectual + * and technical concepts contained herein are proprietary to Adobe + * and its suppliers and are protected by all applicable intellectual + * property laws, including trade secret and copyright laws. + * Dissemination of this information or reproduction of this material + * is strictly forbidden unless prior written permission is obtained + * from Adobe. + ************************************************************************* */ +/* eslint-disable global-require */ +jest.mock('node-fetch', () => require('fetch-mock-jest').sandbox()); +const fetchMock = require('node-fetch'); +const UrlInfo = require('../actions/urlInfo'); + +describe('HelixUtils', () => { + let HelixUtils; + let helixUtils; + let appConfigMock; + + beforeAll(() => { + jest.mock('../actions/utils', () => ({ + getAioLogger: () => ({ + info: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + }), + })); + + appConfigMock = { + getUrlInfo: jest.fn().mockReturnValue( + new UrlInfo('https://example.com/admin?project=p&referrer=https://sp/&owner=o&repo=rp&ref=main') + ), + getConfig: jest.fn().mockReturnValue({ + maxBulkPreviewChecks: 2, + helixAdminApiKeys: { rp: 'key', 'rp-pink': 'key-pink' }, + enablePreviewPublish: ['rp', 'rp-pink'] + }), + }; + HelixUtils = require('../actions/helixUtils'); + helixUtils = new HelixUtils(appConfigMock); + }); + + afterAll(() => { + jest.clearAllMocks(); + }); + + it('should return PREVIEW and LIVE', () => { + const operations = helixUtils.getOperations(); + expect(operations).toEqual({ PREVIEW: 'preview', LIVE: 'live' }); + }); + + it('should return repo without floodgate', () => { + const repo = helixUtils.getRepo(); + expect(repo).toBe('rp'); + }); + + it('should return repo with floodgate', () => { + const repo = helixUtils.getRepo(true, 'blue'); + expect(repo).toBe('rp-blue'); + }); + + it('should return admin api key without floodgate', () => { + const adminApiKey = helixUtils.getAdminApiKey(); + expect(adminApiKey).toBe('key'); + }); + + it('should return admin api key with floodgate', () => { + const adminApiKey = helixUtils.getAdminApiKey(true, 'pink'); + expect(adminApiKey).toBe('key-pink'); + }); + + it('should return true if preview is enabled', () => { + const canPreview = helixUtils.canBulkPreviewPublish(); + expect(canPreview).toBeTruthy(); + }); + + it('submits bulk preview and publish request', async () => { + fetchMock.post('*', () => ({ + messageId: 'a', + job: { + name: 'JN', + } + })); + fetchMock.get('*', () => ({ + progress: 'stopped', + data: { + resources: [ + { path: '/a', status: 200 }, + { path: '/b', status: 200 }, + ], + } + })); + const resp = await helixUtils.bulkPreviewPublish( + ['/a', '/b'], + 'preivew', + { isFloodgate: false }, + 1 + ); + expect(resp).toEqual([{ path: '/a', success: true }, + { path: '/b', success: true }]); + }); +}); diff --git a/test/project.test.js b/test/project.test.js new file mode 100644 index 0000000..5580707 --- /dev/null +++ b/test/project.test.js @@ -0,0 +1,63 @@ +/* ************************************************************************ + * ADOBE CONFIDENTIAL + * ___________________ + * + * Copyright 2023 Adobe + * All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of Adobe and its suppliers, if any. The intellectual + * and technical concepts contained herein are proprietary to Adobe + * and its suppliers and are protected by all applicable intellectual + * property laws, including trade secret and copyright laws. + * Dissemination of this information or reproduction of this material + * is strictly forbidden unless prior written permission is obtained + * from Adobe. + ************************************************************************* */ +describe('project', () => { + let Project = null; + + beforeAll(() => { + jest.mock('../actions/utils', () => ({ + ...jest.requireActual('../actions/utils'), + getAioLogger: () => ({ + info: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + }), + })); + Project = require('../actions/project'); + }); + + it('Get Project Details', async () => { + const project = new Project({ + sharepoint: { + getExcelTable: () => ([[['http://localhost/one']], [['http://localhost/two']]]) + } + }); + const docs = await project.getProjectDetails(); + const entries = docs.urls.entries(); + let entry = entries.next(); + expect(entry.value[0][0]).toBe('http://localhost/one'); + expect(entry.value[1].doc.filePath).toBe('/one.docx'); + entry = entries.next(); + expect(entry.value[0][0]).toBe('http://localhost/two'); + expect(entry.value[1].doc.filePath).toBe('/two.docx'); + }); + + it('Inject Sharepoint Data', () => { + const project = new Project({}); + const projectUrls = new Map().set('http://localhost/doc1', { doc: { sp: { status: 'DONE' } } }); + const filePaths = new Map().set('/doc1', ['http://localhost/doc1']); + project.injectSharepointData(projectUrls, filePaths, ['/doc1'], [{ fileSize: 10 }]); + expect(projectUrls.get('http://localhost/doc1').doc.sp.fileSize).toBe(10); + }); + + it('Update projects with docs', async () => { + const project = new Project({}); + let errs = 0; + try { await project.updateProjectWithDocs(); } catch (err) { errs += 1; } + try { await project.updateProjectWithDocs({}); } catch (err) { errs += 1; } + expect(errs).toBe(2); + }); +}); diff --git a/test/sharepoint.test.js b/test/sharepoint.test.js new file mode 100644 index 0000000..052dab5 --- /dev/null +++ b/test/sharepoint.test.js @@ -0,0 +1,570 @@ +/* ************************************************************************ + * ADOBE CONFIDENTIAL + * ___________________ + * + * Copyright 2023 Adobe + * All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of Adobe and its suppliers, if any. The intellectual + * and technical concepts contained herein are proprietary to Adobe + * and its suppliers and are protected by all applicable intellectual + * property laws, including trade secret and copyright laws. + * Dissemination of this information or reproduction of this material + * is strictly forbidden unless prior written permission is obtained + * from Adobe. + ************************************************************************* */ +const { Headers } = require('node-fetch'); +/* eslint-disable global-require */ +jest.mock('node-fetch', () => require('fetch-mock-jest').sandbox()); +const fetchMock = require('node-fetch'); + +describe('sharepoint', () => { + let Sharepoint = null; + const baseURI = 'https://graph.microsoft.com/v1.0/sites/site.sharepoint.com,d21/drives/123/root:/milo'; + const fgBaseURI = 'https://graph.microsoft.com/v1.0/sites/site.sharepoint.com,d21/drives/123/root:/milo-pink'; + + const appConfig = { + getMsalConfig: () => ({ + clientId: 'CLIENT_ID', + tenantId: 'TENANT_ID', + certThumbprint: 'CERT_THUMB_PRINT', + pvtKey: 'PRIVATE_KEY', + }), + getFgSite: () => 'https://graph.microsoft.com/v1.0/sites/site.sharepoint.com,d21', + getSpConfig: () => ({ + api: { + directory: { + create: { + baseURI, + fgBaseURI, + method: 'PATCH', + payload: { folder: {} }, + }, + }, + file: { + get: { + baseURI, + fgBaseURI, + }, + copy: { + baseURI, + fgBaseURI, + method: 'POST', + payload: { '@microsoft.graph.conflictBehavior': 'replace' }, + }, + createUploadSession: { + baseURI, + fgBaseURI, + method: 'POST', + payload: { '@microsoft.graph.conflictBehavior': 'replace' }, + }, + delete: { + fgBaseURI + }, + update: { + fgBaseURI, + } + }, + excel: { + get: { baseItemsURI: 'https://gql/base' }, + update: { baseItemsURI: 'https://gql/base', method: 'POST' } + } + } + }), + getConfig: () => ({ + fgDirPattern: 'pink' + }) + }; + + beforeAll(() => { + jest.mock('../actions/utils', () => ({ + getAioLogger: () => ({ + info: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + }), + })); + + jest.mock('../actions/sharepointAuth', () => ( + jest.fn().mockReturnValue({ + getAccessToken: jest.fn().mockResolvedValue('AT'), + }))); + Sharepoint = require('../actions/sharepoint'); + }); + + afterAll(() => { + jest.clearAllMocks(); + }); + + // The 'getAuthorizedRequestOption' method returns the authorized request options with the correct headers and access token. + it('should return authorized request options with correct headers and access token', async () => { + const sharepoint = new Sharepoint(appConfig); + const options = await sharepoint.getAuthorizedRequestOption(); + expect(options.headers.get('Authorization')).toContain('Bearer AT'); + }); + + // The 'executeGQL' method throws an error if the response is not ok. + it('should throw an error if the response is not ok', async () => { + const sharepoint = new Sharepoint(appConfig); + sharepoint.fetchWithRetry = jest.fn().mockResolvedValue({ + ok: false + }); + await expect(sharepoint.executeGQL('url', {})).rejects.toThrowError( + 'Failed to execute url' + ); + }); + + // The 'getItemId' method throws an error if the item ID is not found. + it('should get item from path', async () => { + const sharepoint = new Sharepoint(appConfig); + sharepoint.executeGQL = jest.fn().mockResolvedValue({ + id: 10 + }); + const id = await sharepoint.getItemId( + `${appConfig.getFgSite()}/driveid:root`, + '/draft/fg.xlsx' + ); + expect(id).toEqual(10); + }); + + // The 'getDriveRoot' method logs an error if the response is not ok. + it('should log an error if the response is not ok', async () => { + const sharepoint = new Sharepoint(appConfig); + sharepoint.fetchWithRetry = jest + .fn() + .mockResolvedValue({ + ok: false, + status: 500 + }); + await sharepoint.getDriveRoot('accessToken'); + }); + + it('should return an object with fileDownloadUrl and fileSize properties when given a valid file path and isFloodgate boolean', async () => { + // Mock dependencies + const sharepoint = new Sharepoint(appConfig); + const filePath = '/path/to/file.txt'; + const isFloodgate = false; + + // Mock fetchWithRetry function + sharepoint.fetchWithRetry = jest.fn().mockResolvedValue({ + ok: true, + json: jest.fn().mockResolvedValue({ + '@microsoft.graph.downloadUrl': 'https://example.com/file.txt', + size: 1024, + }), + }); + + // Invoke getFileData method + const fileData = await sharepoint.getFileData(filePath, isFloodgate); + + // Assertions + expect(fileData).toEqual({ + fileDownloadUrl: 'https://example.com/file.txt', + fileSize: 1024, + }); + }); + + it('should create a folder in SharePoint when valid folder path is provided', async () => { + // Mock dependencies + const sharepoint = new Sharepoint(appConfig); + sharepoint.getAuthorizedRequestOption = jest.fn().mockResolvedValue({ + method: 'POST', + headers: new Headers(), + body: JSON.stringify({}), + }); + sharepoint.fetchWithRetry = jest.fn().mockResolvedValue({ + ok: true, + json: jest.fn().mockResolvedValue({ id: '12345' }), + }); + + // Test the method + const folder = '/example-folder'; + const isFloodgate = false; + const response = await sharepoint.createFolder(folder, isFloodgate); + + // Assertions + expect(response).toEqual({ id: '12345' }); + expect(sharepoint.getAuthorizedRequestOption).toHaveBeenCalledWith({ + method: 'PATCH', + }); + expect(sharepoint.fetchWithRetry).toHaveBeenCalledWith( + 'https://graph.microsoft.com/v1.0/sites/site.sharepoint.com,d21/drives/123/root:/milo/example-folder', + expect.any(Object) + ); + }); + + it('should return a blob object when given a valid download URL and authorized request options', async () => { + // Mock the necessary dependencies + const sharepoint = new Sharepoint(appConfig); + const downloadUrl = '/file/download'; + + // Mock the fetchWithRetry method + sharepoint.fetchWithRetry = jest.fn().mockResolvedValue({ blob: () => 'Test' }); + + // Invoke the method and assert the result + const result = await sharepoint.getFileUsingDownloadUrl(downloadUrl); + expect(result).toBe('Test'); + }); + + it('should return the file name when given a valid file path with multiple directory levels', () => { + const sharepoint = new Sharepoint(appConfig); + const path = '/folder/subfolder/file.txt'; + const fileName = sharepoint.getFileNameFromPath(path); + expect(fileName).toBe('file.txt'); + }); + + it('should create an upload session with valid inputs', async () => { + // Mock dependencies + const spConfig = { + api: { + file: { + createUploadSession: { + payload: { + // payload properties + }, + method: 'POST', + baseURI: 'https://example.com/api/file', + fgBaseURI: 'https://example.com/api/file/fg', + }, + }, + }, + }; + + const file = { + size: 1024, // file size in bytes + // other file properties + }; + + const dest = '/path/to/destination'; + const filename = 'example.txt'; + const isFloodgate = false; + + const sharepoint = new Sharepoint(appConfig); + sharepoint.getAuthorizedRequestOption = jest.fn().mockResolvedValue({ + method: 'POST', + headers: new Headers(), + body: JSON.stringify({}), + }); + + sharepoint.fetchWithRetry = jest.fn().mockResolvedValue({ + ok: true, + json: jest.fn().mockResolvedValue({ id: '12345' }), + }); + + const uploadSession = await sharepoint.createUploadSession(spConfig, file, dest, filename, isFloodgate); + + // Assertions + expect(uploadSession).toBeDefined(); + // Add more assertions as needed + }); + + it('should upload file successfully when given valid parameters', async () => { + // Mock dependencies + const spConfig = { + api: { + file: { + upload: { + method: 'POST' + } + } + } + }; + const uploadUrl = 'https://example.com/upload'; + const options = { + method: spConfig.api.file.upload.method, + headers: { append: jest.fn() }, + body: {} + }; + const response = { ok: true, json: jest.fn() }; + const sharepoint = new Sharepoint(appConfig); + sharepoint.getAuthorizedRequestOption = jest.fn().mockResolvedValue(options); + sharepoint.fetchWithRetry = jest.fn().mockResolvedValue(response); + + // Invoke method + const result = await sharepoint.uploadFile(spConfig, uploadUrl, {}); + + // Assertions + expect(sharepoint.getAuthorizedRequestOption).toHaveBeenCalledWith({ + json: false, + method: spConfig.api.file.upload.method + }); + expect(sharepoint.fetchWithRetry).toHaveBeenCalledWith(uploadUrl, options); + expect(result).toEqual(response); + }); + + it('should delete file when given valid file path and SharePoint object', async () => { + fetchMock.deleteAny(200); + const sharepoint = new Sharepoint(appConfig); + const sp = { + api: { + file: { + delete: { + method: 'DELETE' + } + } + } + }; + + const filePath = 'https://sp.domain/path/to/file.txt'; + + await sharepoint.releaseUploadSession(sp, filePath); + const response = await sharepoint.deleteFile(sp, filePath); + + expect(response.ok).toBe(true); + }); + it('should rename a file successfully with valid inputs', async () => { + // Mock the necessary dependencies + fetchMock.patchAny(200); + const sharepoint = new Sharepoint(appConfig); + + // Define the test inputs + const spFileUrl = 'https://example.sharepoint.com/sites/testsite/documents/example.docx'; + const filename = 'new_example.docx'; + + // Invoke the method and assert the result + const response = await sharepoint.renameFile(spFileUrl, filename); + expect(response.status).toBe(200); + }); + + it('should return a string with the original filename, a "-locked-" string, and the current timestamp', () => { + const sharepoint = new Sharepoint(appConfig); + const filename = 'example.txt'; + const lockedFileName = sharepoint.getLockedFileNewName(filename); + const regex = /^example-locked-\d+\.txt$/; + expect(regex.test(lockedFileName)).toBe(true); + }); + + it('should create an upload session and upload the file successfully', async () => { + const spConfig = { + api: { + file: { + createUploadSession: { + payload: { + description: 'Preview file', + fileSize: 1000, + name: 'example.txt' + }, + method: 'POST', + baseURI: 'https://example.com/api/files/', + fgBaseURI: 'https://example.com/api/files/floodgate/' + }, + upload: { + method: 'PUT' + } + } + } + }; + + const dest = 'path/to/destination'; + const filename = 'example.txt'; + const isFloodgate = false; + + const sharepoint = new Sharepoint(appConfig); + const response = { ok: true, json: jest.fn().mockReturnValue({ uploadUrl: 'url' }) }; + sharepoint.fetchWithRetry = jest.fn().mockResolvedValue(response); + const result = await sharepoint.createSessionAndUploadFile(spConfig, {}, dest, filename, isFloodgate); + + expect(result.success).toBe(true); + expect(result.uploadedFile).toBeDefined(); + }); + + it('should create folders for a list of file paths successfully', async () => { + // Mock dependencies + const sharepointAuth = jest.fn(); + const getAccessToken = jest.fn().mockResolvedValue('token'); + const getAuthorizedRequestOption = jest.fn().mockResolvedValue({}); + const executeGQL = jest.fn().mockResolvedValue({ ok: true }); + const createFolder = jest.fn().mockResolvedValue({ ok: true }); + + // Initialize Sharepoint class object + const sharepoint = new Sharepoint(appConfig); + const response = { ok: true, json: jest.fn().mockReturnValue({ uploadUrl: 'url' }) }; + sharepoint.sharepointAuth = sharepointAuth; + sharepoint.sharepointAuth.getAccessToken = getAccessToken; + sharepoint.getAuthorizedRequestOption = getAuthorizedRequestOption; + sharepoint.executeGQL = executeGQL; + sharepoint.createFolder = createFolder; + sharepoint.fetchWithRetry = jest.fn().mockResolvedValue(response); + + // Define input + const srcPathList = [ + [1, { doc: { filePath: '/a/b/one.txt' } }], + [2, { doc: { filePath: '/a/b/two.txt' } }], + [3, { doc: { filePath: '/a/c/three.txt' } }], + [4, { doc: { filePath: '/a/c/d/three.txt' } }] + ]; + const isFloodgate = false; + + // Invoke method + const result = await sharepoint.bulkCreateFolders(srcPathList, isFloodgate); + + // Assertions + expect(result).toEqual([{ ok: true }, { ok: true }]); + expect(createFolder).toHaveBeenCalledTimes(2); + expect(createFolder).toHaveBeenCalledWith('/a/b', isFloodgate); + expect(createFolder).toHaveBeenCalledWith('/a/c/d', isFloodgate); + }); + + it('should copy a file from source path to destination folder', async () => { + const sharepoint = new Sharepoint(appConfig); + sharepoint.fetchWithRetry = jest.fn().mockResolvedValue({ ok: true, json: () => ({ status: 'completed' }), headers: { get: jest.fn().mockReturnValue({ Location: 'https://sp/file' }) } }); + + // Test input + const srcPath = '/path/to/source/file.txt'; + const destinationFolder = '/path/to/destination/folder'; + const newName = null; + const isFloodgate = false; + const isFloodgateLockedFile = false; + + // Invoke method + const copySuccess = await sharepoint.copyFile(srcPath, destinationFolder, newName, isFloodgate, isFloodgateLockedFile); + + // Assertions + expect(copySuccess).toBe(true); + }); + + it('should save', async () => { + // Mock dependencies + const sharepoint = new Sharepoint(appConfig); + const folder = '/Documents'; + const filename = 'example.txt'; + const isFloodgate = false; + sharepoint.getAuthorizedRequestOption = jest.fn().mockResolvedValue({ + method: 'POST', + headers: new Headers(), + body: JSON.stringify({}), + }); + jest.spyOn(sharepoint, 'createFolder').mockResolvedValueOnce({}); + jest.spyOn(sharepoint, 'createSessionAndUploadFile').mockResolvedValueOnce({ locked: true }).mockResolvedValueOnce({ success: true, uploadedFile: {} }); + jest.spyOn(sharepoint, 'releaseUploadSession').mockResolvedValueOnce({}); + jest.spyOn(sharepoint, 'getLockedFileNewName').mockResolvedValueOnce(filename); + jest.spyOn(sharepoint, 'renameFile').mockResolvedValueOnce(filename); + jest.spyOn(sharepoint, 'copyFile').mockResolvedValueOnce({}); + jest.spyOn(sharepoint, 'deleteFile').mockResolvedValueOnce({}); + + // Invoke the method + const resp = await sharepoint.saveFile({}, `${folder}/${filename}`, isFloodgate); + + // Verify the behavior + expect(resp.success).toBe(true); + }); + + it('should return a list of rows when tableJson value is truthy', async () => { + const sharepoint = new Sharepoint(appConfig); + jest.spyOn(sharepoint, 'getItemId').mockResolvedValueOnce('itemId'); + jest.spyOn(sharepoint, 'executeGQL').mockResolvedValueOnce({ value: [{ values: [['data']] }] }); + + const excelPath = '/path/to/excel/file.xlsx'; + const tableName = 'Table1'; + + const result = await sharepoint.getExcelTable(excelPath, tableName); + + expect(result).toEqual([[['data']]]); + }); + + it('should delete the specified folder', async () => { + const sharepoint = new Sharepoint({ ...appConfig, fgDirPattern: 'test' }); + jest.spyOn(sharepoint, 'deleteFile').mockResolvedValueOnce({}); + const deleteSuccess = await sharepoint.deleteFloodgateDir(); + + expect(deleteSuccess).toBe(true); + }); + + it('should update the table when the Excel file and table exist', async () => { + const sharepoint = new Sharepoint(appConfig); + jest.spyOn(sharepoint, 'getItemId').mockResolvedValueOnce('itemId'); + jest.spyOn(sharepoint, 'executeGQL').mockResolvedValueOnce({}); + + // Test code + const excelPath = 'path/to/excel/file.xlsx'; + const tableName = 'Sheet1'; + const values = [ + ['A1', 'B1', 'C1'], + ['A2', 'B2', 'C2'], + ['A3', 'B3', 'C3'], + ]; + + const response = await sharepoint.updateExcelTable(excelPath, tableName, values); + + expect(sharepoint.getItemId).toHaveBeenCalledWith(baseURI, excelPath); + expect(sharepoint.executeGQL).toHaveBeenCalledWith('https://gql/base/itemId/workbook/tables/Sheet1/rows', { + body: JSON.stringify({ values }), + method: 'POST', + }); + expect(response).toEqual({}); + }); + + it('should handle headers with duplicate names by overwriting the previous value with the new one', () => { + const response = { + headers: new Map([ + ['Content-Type', 'application/json'], + ['Authorization', 'Bearer token123'], + ['User-Agent', 'MyApp'], + ['Content-Type', 'text/plain'], + ]), + status: 200, + }; + + const sharepoint = new Sharepoint(appConfig); + const headersStr = sharepoint.getHeadersStr(response); + + expect(headersStr).toBe('{"Content-Type":"text/plain","Authorization":"Bearer token123","User-Agent":"MyApp"}'); + }); + + it('should log response status and headers when getLogRespHeader is true', () => { + const response = { + status: 200, + headers: new Headers({ + 'Content-Type': 'application/json', + 'RateLimit-Reset': '3600', + 'Retry-After': '120', + }), + }; + + const sharepoint = new Sharepoint(appConfig); + jest.spyOn(sharepoint, 'getLogRespHeader').mockResolvedValueOnce(true); + jest.spyOn(sharepoint, 'getHeadersStr').mockResolvedValueOnce('Content-Type: application/json'); + + expect(() => sharepoint.logHeaders(response)).not.toThrow(); + }); + + // Handles rate limit headers and 429 errors by retrying the request after the specified time + it('should handle rate limit headers and 429 errors by retrying the request after the specified time', async () => { + // Mock the fetch function to return a response with rate limit headers or status code 429 + const mockResponse = { + status: 429, + headers: { + 'ratelimit-reset': '1', + 'retry-after': '1', + 'test-retry-status': '429' + } + }; + const mockResponse2 = { + status: 200, + body: {} + }; + let fetchMockCalled = 0; + fetchMock.mock('*', () => { + fetchMockCalled += 1; + return fetchMockCalled > 1 ? mockResponse2 : mockResponse; + }); + const apiUrl = 'https://api.example.com/data'; + const options = { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer token123', + 'test-retry-status': 'true' + } + }; + + const sharepoint = new Sharepoint(appConfig); + const response = await sharepoint.fetchWithRetry(apiUrl, options); + + expect(response).toBeDefined(); + expect(response.status).toBe(200); + const data = await response.json(); + expect(data).toEqual({}); + }); +}); diff --git a/test/sharepointAuth.test.js b/test/sharepointAuth.test.js new file mode 100644 index 0000000..e2dd903 --- /dev/null +++ b/test/sharepointAuth.test.js @@ -0,0 +1,92 @@ +/* ************************************************************************ + * ADOBE CONFIDENTIAL + * ___________________ + * + * Copyright 2023 Adobe + * All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of Adobe and its suppliers, if any. The intellectual + * and technical concepts contained herein are proprietary to Adobe + * and its suppliers and are protected by all applicable intellectual + * property laws, including trade secret and copyright laws. + * Dissemination of this information or reproduction of this material + * is strictly forbidden unless prior written permission is obtained + * from Adobe. + ************************************************************************* */ + +describe('sharepointAuth', () => { + let SharepointAuth = null; + const testTknStr = + 'ZXlKaGJHY2lPaUpJVXpJMU5pSXNJblI1Y0NJNklrcFhWQ0o5LmV5SmhkV1FpT2lJd01EQXd' + + 'NREF3TVMwd01EQXdMVEF3TURBdE1EQXdNQzB3TURBd01EQXdNREF3TURBaUxDSnBjM01p' + + 'T2lKb2RIUndjem92TDNOMGN5NTNhVzVrYjNkekxtNWxkQzh4TVRFeE1URXhMVEV4TVRFdE' + + '1URXhNUzB4TVRFeExURXhNVEV4TVRFeE1URXhNUzhpTENKaGNIQmZaR2x6Y0d4aGVXNWhi' + + 'V1VpT2lKSGNtRndhQ0JGZUhCc2IzSmxjaUlzSW1Gd2NHbGtJam9pTWpJeU1qSXlNaTB4TV' + + 'RFeExURXhNVEV0TVRFeE1TMHhNVEV4TVRFeE1URXhNVEVpTENKaGNIQnBaR0ZqY2lJNklq' + + 'QWlMQ0ptWVcxcGJIbGZibUZ0WlNJNklrRWlMQ0puYVhabGJsOXVZVzFsSWpvaVZYTmxjaU' + + 'lzSW1sa2RIbHdJam9pZFhObGNpSXNJbTVoYldVaU9pSlZjMlZ5SUVFaUxDSnZhV1FpT2lJ' + + 'd01EQXdNREF4TFRFeE1URXRNVEV4TVMweE1URXhMVEV4TVRFeE1URXhNVEV4TVNJc0ltOX' + + 'VjSEpsYlY5emFXUWlPaUpUTFRFdE5TMHlNU0lzSW5Cc1lYUm1Jam9pTlNJc0luQjFhV1Fp' + + 'T2lJeE1URXhJaXdpZFc1cGNYVmxYMjVoYldVaU9pSjFjMlZ5WVVCemIyMWxaRzl0WVdsdU' + + 'xtTnZiU0lzSW5Wd2JpSTZJblZ6WlhKaFFITnZiV1ZrYjIxaGFXNHVZMjl0SW4wLllTN0l5' + + 'cVFEUlFMYWRWQjNMTHNLQTRhWXFpSkptZGdUQ1VRNXlNVmFTUkE='; + const mockTestTkn = Buffer.from(testTknStr, 'base64').toString(); + + const msalConfig = { + clientId: 'CLIENT_ID', + tenantId: 'TENANT_ID', + certThumbprint: 'CERT_THUMB_PRINT', + pvtKey: 'PRIVATE_KEY' + }; + + beforeAll(() => { + jest.mock('../actions/utils', () => ({ + getAioLogger: () => ({ + info: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + }), + })); + + jest.mock('@azure/msal-node', () => ({ + ConfidentialClientApplication: jest.fn().mockReturnValue({ + acquireTokenByClientCredential: jest.fn().mockResolvedValue({ + accessToken: mockTestTkn + }) + }) + })); + SharepointAuth = require('../actions/sharepointAuth'); + }); + + afterAll(() => { + jest.clearAllMocks(); + }); + + it('should extract user details', async () => { + const sharepointAuth = new SharepointAuth(msalConfig); + const at = await sharepointAuth.getAccessToken(); + expect(at).toEqual(mockTestTkn); + const ud = await sharepointAuth.getUserDetails(at); + expect(ud.oid).toEqual('0000001-1111-1111-1111-111111111111'); + }); + + it('missing parameters in config', async () => { + let uddMsalConfig = { ...msalConfig, clientId: '' }; + const errChk = /mandatory fields have not been configured/; + expect(() => new SharepointAuth(uddMsalConfig)).toThrow(errChk); + uddMsalConfig = { ...msalConfig, tenantId: '' }; + expect(() => new SharepointAuth(uddMsalConfig)).toThrow(errChk); + uddMsalConfig = { ...msalConfig, certThumbprint: '' }; + expect(() => new SharepointAuth(uddMsalConfig)).toThrow(errChk); + uddMsalConfig = { ...msalConfig, pvtKey: '' }; + expect(() => new SharepointAuth(uddMsalConfig)).toThrow(errChk); + }); + + it('should check the token expiry', async () => { + const sharepointAuth = new SharepointAuth(msalConfig); + const at = await sharepointAuth.getAccessToken(); + const exp = sharepointAuth.isTokenExpired(at); + expect(exp).toBe(true); + }); +}); diff --git a/test/urlinfo.test.js b/test/urlinfo.test.js new file mode 100644 index 0000000..755cc12 --- /dev/null +++ b/test/urlinfo.test.js @@ -0,0 +1,52 @@ +/* ************************************************************************ +* ADOBE CONFIDENTIAL +* ___________________ +* +* Copyright 2023 Adobe +* All Rights Reserved. +* +* NOTICE: All information contained herein is, and remains +* the property of Adobe and its suppliers, if any. The intellectual +* and technical concepts contained herein are proprietary to Adobe +* and its suppliers and are protected by all applicable intellectual +* property laws, including trade secret and copyright laws. +* Dissemination of this information or reproduction of this material +* is strictly forbidden unless prior written permission is obtained +* from Adobe. +************************************************************************* */ +const UrlInfo = require('../actions/urlInfo'); + +describe('UrlInfo', () => { + // Constructing a UrlInfo object with a valid adminPageUri sets the urlInfoMap with the correct values. + it('should set urlInfoMap with correct values when adminPageUri is valid', () => { + const adminPageUri = 'https://example.com/admin?project=projectName&referrer=referrerName&owner=ownerName&repo=repoName&ref=branchName'; + const urlInfo = new UrlInfo(adminPageUri); + + expect(urlInfo.getUrlInfo()).toEqual({ + sp: 'referrerName', + owner: 'ownerName', + repo: 'repoName', + branch: 'branchName', + origin: 'https://branchName--repoName--ownerName.hlx.page' + }); + }); + + // Constructing a UrlInfo object with an invalid adminPageUri sets the urlInfoMap with undefined values. + it('should set urlInfoMap with values', () => { + const adminPageUri = 'https://example.com/admin?project=p&referrer=https://sp/&owner=o&repo=rp&ref=main'; + const urlInfo = new UrlInfo(adminPageUri); + + expect(urlInfo.getUrlInfo()).toEqual({ + sp: 'https://sp/', + owner: 'o', + repo: 'rp', + branch: 'main', + origin: 'https://main--rp--o.hlx.page' + }); + + expect(urlInfo.getOrigin()).toEqual('https://main--rp--o.hlx.page'); + expect(urlInfo.getBranch()).toEqual('main'); + expect(urlInfo.getOwner()).toEqual('o'); + expect(urlInfo.getRepo()).toEqual('rp'); + }); +}); diff --git a/test/utils.test.js b/test/utils.test.js index 0e6db1f..22b8964 100644 --- a/test/utils.test.js +++ b/test/utils.test.js @@ -14,143 +14,184 @@ * is strictly forbidden unless prior written permission is obtained * from Adobe. ************************************************************************* */ +const events = require('events'); -const utils = require('../actions/utils'); +describe('utils', () => { + let utils = null; -test('interface', () => { - expect(typeof utils.handleExtension).toBe('function'); -}); - -describe('handleExtension', () => { - test('docx path', () => { - expect(utils.handleExtension('/path/to/file.docx')).toEqual('/path/to/file'); - }); - test('xlsx path', () => { - expect(utils.handleExtension('/path/to/file.xlsx')).toEqual('/path/to/file.json'); - }); - test('svg path', () => { - expect(utils.handleExtension('/path/to/file.svg')).toEqual('/path/to/file.svg'); - }); - test('docx path', () => { - expect(utils.handleExtension('/path/to/index.docx')).toEqual('/path/to/'); - }); - test('files with caps in path', () => { - expect(utils.handleExtension('/path/to/Sample.docx')).toEqual('/path/to/sample'); - }); - test('files with space in path', () => { - expect(utils.handleExtension('/path/to/sample test.docx')).toEqual('/path/to/sample-test'); + beforeAll(() => { + jest.mock('@adobe/aio-lib-core-logging', () => (() => ({ + info: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + }))); + utils = require('../actions/utils'); }); - test('files with space in path', () => { - expect(utils.handleExtension('/path/to/Sample_Test.docx')).toEqual('/path/to/sample-test'); - }); - test('file in root path', () => { - expect(utils.handleExtension('/Sample_Test.docx')).toEqual('/sample-test'); - }); - test('file without extension', () => { - expect(utils.handleExtension('/Sample_Test')).toEqual('/sample-test'); - }); -}); -describe('strToArray', () => { - const td1 = ['a', 'b', 'c']; - test('str to array', () => { - expect(utils.strToArray('a,b,c')).toEqual(td1); + afterAll(() => { + jest.clearAllMocks(); }); - test('str to array with array input', () => { - expect(utils.strToArray(td1)).toEqual(td1); - }); -}); -describe('toUTCStr', () => { - test('iso string', () => { - expect(utils.toUTCStr('2023-11-07T07:00:33.462Z')).toEqual('Tue, 07 Nov 2023 07:00:33 GMT'); - }); - test('iso date', () => { - expect(utils.toUTCStr(new Date('2023-11-07T07:00:33.462Z'))).toEqual('Tue, 07 Nov 2023 07:00:33 GMT'); + describe('handleExtension', () => { + test('docx path', () => { + expect(utils.handleExtension('/path/to/file.docx')).toEqual('/path/to/file'); + }); + test('xlsx path', () => { + expect(utils.handleExtension('/path/to/file.xlsx')).toEqual('/path/to/file.json'); + }); + test('svg path', () => { + expect(utils.handleExtension('/path/to/file.svg')).toEqual('/path/to/file.svg'); + }); + test('docx path', () => { + expect(utils.handleExtension('/path/to/index.docx')).toEqual('/path/to/'); + }); + test('files with caps in path', () => { + expect(utils.handleExtension('/path/to/Sample.docx')).toEqual('/path/to/sample'); + }); + test('files with space in path', () => { + expect(utils.handleExtension('/path/to/sample test.docx')).toEqual('/path/to/sample-test'); + }); + test('files with space in path', () => { + expect(utils.handleExtension('/path/to/Sample_Test.docx')).toEqual('/path/to/sample-test'); + }); + test('file in root path', () => { + expect(utils.handleExtension('/Sample_Test.docx')).toEqual('/sample-test'); + }); + test('file without extension', () => { + expect(utils.handleExtension('/Sample_Test')).toEqual('/sample-test'); + }); }); - test('utc string', () => { - expect(utils.toUTCStr('Tue, 07 Nov 2023 07:00:33 GMT')).toEqual('Tue, 07 Nov 2023 07:00:33 GMT'); - }); - test('another string', () => { - expect(utils.toUTCStr('2023-NOV-07 07:00:33 AM GMT')).toEqual('Tue, 07 Nov 2023 07:00:33 GMT'); - }); - test('empty string', () => { - expect(utils.toUTCStr('')).toEqual(''); + + describe('strToArray', () => { + const td1 = ['a', 'b', 'c']; + test('str to array', () => { + expect(utils.strToArray('a,b,c')).toEqual(td1); + }); + test('str to array with array input', () => { + expect(utils.strToArray(td1)).toEqual(td1); + }); }); - test('no val', () => { - expect(utils.toUTCStr()).toEqual(undefined); + + describe('toUTCStr', () => { + test('iso string', () => { + expect(utils.toUTCStr('2023-11-07T07:00:33.462Z')).toEqual('Tue, 07 Nov 2023 07:00:33 GMT'); + }); + test('iso date', () => { + expect(utils.toUTCStr(new Date('2023-11-07T07:00:33.462Z'))).toEqual('Tue, 07 Nov 2023 07:00:33 GMT'); + }); + test('utc string', () => { + expect(utils.toUTCStr('Tue, 07 Nov 2023 07:00:33 GMT')).toEqual('Tue, 07 Nov 2023 07:00:33 GMT'); + }); + test('another string', () => { + expect(utils.toUTCStr('2023-NOV-07 07:00:33 AM GMT')).toEqual('Tue, 07 Nov 2023 07:00:33 GMT'); + }); + test('empty string', () => { + expect(utils.toUTCStr('')).toEqual(''); + }); + test('no val', () => { + expect(utils.toUTCStr()).toEqual(undefined); + }); }); -}); -describe('isFilePathWithWildcard', () => { - test('matches exact file path', () => { - expect(utils.isFilePathWithWildcard('/path/to/file.txt', '/path/to/file.txt')).toBe(true); + describe('isFilePathWithWildcard', () => { + test('matches exact file path', () => { + expect(utils.isFilePathWithWildcard('/path/to/file.txt', '/path/to/file.txt')).toBe(true); + }); + + test('matches file path with wildcard', () => { + expect(utils.isFilePathWithWildcard('/path/to/directory/', '/path/to/*')).toBe(true); + }); + + test('matches file with wildcard extension', () => { + expect(utils.isFilePathWithWildcard('file_with_space.txt', '*.txt')).toBe(true); + }); + + test('match a prefix wild card', () => { + expect(utils.isFilePathWithWildcard('/drafts/a/query-index.xlsx', '*/query-index.xlsx')).toBe(true); + expect(utils.isFilePathWithWildcard('/drafts/b/query-index.xlsx', '*/query-index.xlsx')).toBe(true); + expect(utils.isFilePathWithWildcard('/drafts/b/c/query-index.xlsx', '*/query-index.xlsx')).toBe(true); + }); + + test('matches dot files', () => { + expect(utils.isFilePathWithWildcard('/.milo', '/.milo')).toBe(true); + expect(utils.isFilePathWithWildcard('/amilo', '/.milo')).toBe(false); + }); }); - test('matches file path with wildcard', () => { - expect(utils.isFilePathWithWildcard('/path/to/directory/', '/path/to/*')).toBe(true); + describe('isFilePatternMatched', () => { + const patterns = ['/.milo', '/.helix', '/metadata.xlsx', '/a/Caps', '*/query-index.xlsx']; + test('matches a set of file', () => { + expect(utils.isFilePatternMatched('/.helix', patterns)).toBe(true); + expect(utils.isFilePatternMatched('/a/Caps', patterns)).toBe(true); + expect(utils.isFilePatternMatched('/a/Caps/Test', patterns)).toBe(true); + expect(utils.isFilePatternMatched('/a/ACaps/Test', patterns)).toBe(false); + expect(utils.isFilePatternMatched('/a/query-index.xlsx', patterns)).toBe(true); + expect(utils.isFilePatternMatched('/a/b/query-index.xlsx', patterns)).toBe(true); + }); }); - test('matches file with wildcard extension', () => { - expect(utils.isFilePathWithWildcard('file_with_space.txt', '*.txt')).toBe(true); + describe('strToBool', () => { + test('Check for a set of true and false inputs', () => { + expect(utils.strToBool('true')).toBeTruthy(); + expect(utils.strToBool('TRUE')).toBeTruthy(); + expect(utils.strToBool(true)).toBeTruthy(); + expect(utils.strToBool('false')).not.toBeTruthy(); + expect(utils.strToBool('FALSE')).not.toBeTruthy(); + expect(utils.strToBool(false)).not.toBeTruthy(); + expect(utils.strToBool('something')).not.toBeTruthy(); + expect(utils.strToBool('')).not.toBeTruthy(); + expect(utils.strToBool(null)).not.toBeTruthy(); + expect(utils.strToBool(undefined)).not.toBeTruthy(); + }); }); - test('match a prefix wild card', () => { - expect(utils.isFilePathWithWildcard('/drafts/a/query-index.xlsx', '*/query-index.xlsx')).toBe(true); - expect(utils.isFilePathWithWildcard('/drafts/b/query-index.xlsx', '*/query-index.xlsx')).toBe(true); - expect(utils.isFilePathWithWildcard('/drafts/b/c/query-index.xlsx', '*/query-index.xlsx')).toBe(true); + describe('getDocPathFromUrl', () => { + test('Check extension handling', () => { + expect(utils.getDocPathFromUrl('https://sp/path/file')).toEqual('/path/file.docx'); + expect(utils.getDocPathFromUrl('https://sp/path/file.json')).toEqual('/path/file.xlsx'); + expect(utils.getDocPathFromUrl('https://sp/path/file.svg')).toEqual('/path/file.svg'); + expect(utils.getDocPathFromUrl('https://sp/path/file.pdf')).toEqual('/path/file.pdf'); + expect(utils.getDocPathFromUrl('https://sp/path/')).toEqual('/path/index.docx'); + expect(utils.getDocPathFromUrl('https://sp/path/file.html')).toEqual('/path/file.docx'); + }); }); - test('matches dot files', () => { - expect(utils.isFilePathWithWildcard('/.milo', '/.milo')).toBe(true); - expect(utils.isFilePathWithWildcard('/amilo', '/.milo')).toBe(false); + describe('getInstanceKey', () => { + test('Check instance key cleanup', () => { + expect(utils.getInstanceKey({ fgRootFolder: 'Clean-Up%20This_123' })).toEqual('Clean_Up_20This_123'); + }); }); -}); -describe('isFilePatternMatched', () => { - const patterns = ['/.milo', '/.helix', '/metadata.xlsx', '/a/Caps', '*/query-index.xlsx']; - test('matches a set of file', () => { - expect(utils.isFilePatternMatched('/.helix', patterns)).toBe(true); - expect(utils.isFilePatternMatched('/a/Caps', patterns)).toBe(true); - expect(utils.isFilePatternMatched('/a/Caps/Test', patterns)).toBe(true); - expect(utils.isFilePatternMatched('/a/ACaps/Test', patterns)).toBe(false); - expect(utils.isFilePatternMatched('/a/query-index.xlsx', patterns)).toBe(true); - expect(utils.isFilePatternMatched('/a/b/query-index.xlsx', patterns)).toBe(true); + describe('logMemUsageIter', () => { + test('Check instance key cleanup', () => { + const onMock = jest.spyOn(events.EventEmitter.prototype, 'on'); + utils.logMemUsageIter(); + expect(onMock).toHaveBeenCalled(); + }); }); -}); -describe('strToBool', () => { - test('Check for a set of true and false inputs', () => { - expect(utils.strToBool('true')).toBeTruthy(); - expect(utils.strToBool('TRUE')).toBeTruthy(); - expect(utils.strToBool(true)).toBeTruthy(); - expect(utils.strToBool('false')).not.toBeTruthy(); - expect(utils.strToBool('FALSE')).not.toBeTruthy(); - expect(utils.strToBool(false)).not.toBeTruthy(); - expect(utils.strToBool('something')).not.toBeTruthy(); - expect(utils.strToBool('')).not.toBeTruthy(); - expect(utils.strToBool(null)).not.toBeTruthy(); - expect(utils.strToBool(undefined)).not.toBeTruthy(); + describe('actInProgress', () => { + test('should return true when svInProg is true, actId is not null, and activation status is null', async () => { + const ow = {}; + const actId = '123'; + const svInProg = true; + const result = await utils.actInProgress(ow, actId, svInProg); + expect(result).toBe(true); + }); + + test('should return false when svInProg is false and actId is provided', async () => { + const ow = {}; + const actId = '123'; + const svInProg = false; + const result = await utils.actInProgress(ow, actId, svInProg); + expect(result).toBe(false); + }); }); -}); -describe('getDocPathFromUrl', () => { - test('Check for a set of true and false inputs', () => { - expect(utils.getDocPathFromUrl('https://main--cc-pink--adobecom.hlx.page/drafts/test/samplepage')) - .toEqual('/drafts/test/samplepage.docx'); - expect(utils.getDocPathFromUrl('https://main--cc-pink--adobecom.hlx.page/drafts/test/')) - .toEqual('/drafts/test/index.docx'); - expect(utils.getDocPathFromUrl('https://main--cc-pink--adobecom.hlx.page/drafts/test/samplesheet.json')) - .toEqual('/drafts/test/samplesheet.xlsx'); - expect(utils.getDocPathFromUrl('https://main--cc-pink--adobecom.hlx.page/drafts/test/samplehtmlpage.html')) - .toEqual('/drafts/test/samplehtmlpage.docx'); - expect(utils.getDocPathFromUrl('https://main--cc-pink--adobecom.hlx.page/drafts/test/samplepdf.pdf')) - .toEqual('/drafts/test/samplepdf.pdf'); - expect(utils.getDocPathFromUrl('https://main--cc-pink--adobecom.hlx.page/drafts/test/samplevimg.svg')) - .toEqual('/drafts/test/samplevimg.svg'); - expect(utils.getDocPathFromUrl('https://main--cc-pink--adobecom.hlx.page/drafts/test/axsizzle-marquee.mp4')) - .toEqual('/drafts/test/axsizzle-marquee.mp4'); - expect(utils.getDocPathFromUrl('https://main--cc-pink--adobecom.hlx.page/drafts/test/axsizzle-marquee.mp4#_autoplay')) - .toEqual('/drafts/test/axsizzle-marquee.mp4'); + describe('errorResponse', () => { + test('error response is built', async () => { + const er = utils.errorResponse(401, 'Auth Error'); + expect(er).toEqual({ error: { statusCode: 401, body: { error: 'Auth Error' } } }); + }); }); });