Skip to content

Commit

Permalink
MWPW-144264 Unpublish content when delete action is run on the floodg…
Browse files Browse the repository at this point in the history
…ate (#105)

### On Delete: Unpublish Content

1. Retrieve the files to unpublish using the Helix status URLs.
2. Batch unpublish the files.
3. After unpublishing, delete the folder.

### Configuration Keys Added:
- `DELETE_ROOT_PATH=/drafts/raga/tmp/`
- `MAX_DELETE_FILES_PER_BATCH=350`

> Set `DELETE_ROOT_PATH` to empty once live and adjust
`MAX_DELETE_FILES_PER_BATCH` as needed for batching.

### State Keys Change (use `stateStore` to fetch)
**Example:**
-
`copyAction:/adobecom/CC/www-pink:/drafts/floodgate/projects/raga/samplefgprojectone.xlsx`
- `promoteAction:/adobecom/CC/www-pink`
- `deleteAction:/adobecom/CC/www-pink`


## TESTING
**Before delete**

![image](https://github.com/user-attachments/assets/0cd3c581-92ca-4c45-b355-9eb0e697b910)

**After Delete**

![image](https://github.com/user-attachments/assets/b5ea9124-ba0e-4ce4-a2be-227622370db2)


**Maintenance Tracker Changes**

![image](https://github.com/user-attachments/assets/1800fddb-c2d5-4219-94e1-0b8a2b26e647)
  • Loading branch information
sukamat authored Oct 4, 2024
2 parents 87cd0bb + 166794f commit 3fb4bed
Show file tree
Hide file tree
Showing 19 changed files with 510 additions and 132 deletions.
24 changes: 24 additions & 0 deletions actions/appConfig.js
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ class AppConfig {
this.configMap.bulkPreviewCheckInterval = parseInt(params.bulkPreviewCheckInterval || '30', 10);
this.configMap.maxBulkPreviewChecks = parseInt(params.maxBulkPreviewChecks || '30', 10);
this.configMap.enablePreviewPublish = this.getJsonFromStr(params.enablePreviewPublish, []);
this.configMap.deleteRootPath = params.deleteRootPath;
this.configMap.maxDeleteFilesPerBatch = params.maxDeleteFilesPerBatch || 100;
this.extractPrivateKey();

payload.ext = {
Expand Down Expand Up @@ -169,6 +171,13 @@ class AppConfig {
};
}

getDeleteBatchConfig() {
return {
batchFilesPath: this.configMap.batchFilesPath,
maxFilesPerBatch: this.configMap.maxDeleteFilesPerBatch,
};
}

getNumBulkReq() {
return this.configMap.numBulkReq;
}
Expand Down Expand Up @@ -283,6 +292,21 @@ class AppConfig {
},
};
}

getFgFolderToDelete() {
const { deleteRootPath, fgDirPattern } = this.getConfig();
const { fgRootFolder } = this.getPayload();
const fgRegExp = new RegExp(fgDirPattern);
const { api: { file: { update: { fgBaseURI } } } } = this.getSpConfig();
if (fgRegExp.test(fgRootFolder)) {
const suffix = (deleteRootPath || '/tmp').replace(/\/+$/, '');
return {
suffix,
url: `${fgBaseURI}${suffix}`,
};
}
return null;
}
}

module.exports = AppConfig;
2 changes: 1 addition & 1 deletion actions/batch.js
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ class Batch {
}

async getFiles() {
logger.info(`get batch files ${this.filesSdk} and ${this.instancePath}`);
logger.info(`get batch files from ${this.instancePath}`);
let fileContents = [];
if (this.filesSdk && this.instancePath) {
const dataStr = await this.filesSdk.read(this.batchInfoFile);
Expand Down
31 changes: 24 additions & 7 deletions actions/batchManager.js
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,13 @@ class BatchManager {
}

/**
* Write to promoteAction/tracker.json
* Write to ___Action/tracker.json
*/
async writeToBmTracker(data) {
await this.filesSdk.write(this.bmTracker, JSON.stringify(data));
}

async updateBmTracker(data) {
const content = await this.readBmTracker();
content.instanceKeys = content.instanceKeys || [];
if (content.instanceKeys) {
Expand All @@ -136,7 +140,22 @@ class BatchManager {
content.instanceKeys.push(this.instanceKey);
}
}
await this.filesSdk.write(this.bmTracker, JSON.stringify({ ...content, ...data }));
await this.writeToBmTracker({ ...content, ...data });
}

async isInstanceRunning() {
const tracker = await this.readBmTracker();
return tracker.running;
}

async markInstanceRunning() {
const tracker = await this.readBmTracker();
await this.writeToBmTracker({ ...tracker, running: new Date().getTime() });
}

async markInstancePaused() {
const { running, ...rest } = await this.readBmTracker();
await this.writeToBmTracker(rest);
}

/**
Expand Down Expand Up @@ -197,21 +216,19 @@ class BatchManager {
async finalizeInstance(addlParams) {
// Save any pending files in the batch
await this.currentBatch?.savePendingFiles();

// If there are any additional parameters then add to the instance file.
if (addlParams) {
const ifc = await this.getInstanceFileContent();
ifc.dtls = { ...ifc.dtls, ...addlParams };
await this.writeToInstanceFile(ifc);
}

// Update promoteAction/tracker.json to start the batch processing
const params = {};
const params = { running: 0 };
params[`${this.instanceKey}`] = {
done: false,
proceed: true,
};
await this.writeToBmTracker(params);
await this.updateBmTracker(params);
}

async markComplete(results) {
Expand All @@ -221,7 +238,7 @@ class BatchManager {
proceed: false,
};
if (results) await this.writeResults(results);
await this.writeToBmTracker(params);
await this.updateBmTracker(params);
}

/**
Expand Down
57 changes: 38 additions & 19 deletions actions/delete/worker.js → actions/delete/createBatch.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,16 @@
* from Adobe.
************************************************************************* */
const openwhisk = require('openwhisk');
const Sharepoint = require('../sharepoint');
const {
toUTCStr, getAioLogger, logMemUsage, DELETE_ACTION
getAioLogger, logMemUsage, getInstanceKey, DELETE_ACTION
} = require('../utils');
const FgStatus = require('../fgStatus');
const FgAction = require('../fgAction');
const AppConfig = require('../appConfig');
const BatchManager = require('../batchManager');
const HelixUtils = require('../helixUtils');
const FgDeleteActionHelper = require('../fgDeleteActionHelper');
const Sharepoint = require('../sharepoint');

async function main(params) {
logMemUsage();
Expand All @@ -32,47 +35,63 @@ async function main(params) {
statParams: ['fgRootFolder', 'projectExcelPath'],
actParams: ['adminPageUri'],
};
const ow = openwhisk();

// Initialize action
const ow = openwhisk();
const appConfig = new AppConfig(params);
const fgAction = new FgAction(DELETE_ACTION, appConfig);
fgAction.init({ ow, skipUserDetails: true });

const { fgStatus } = fgAction.getActionParams();
const { projectExcelPath } = appConfig.getPayload();
const { projectExcelPath, fgColor } = appConfig.getPayload();
const fgDeleteActionHelper = new FgDeleteActionHelper();
const sharepoint = new Sharepoint(appConfig);

try {
// Validations
const vStat = await fgAction.validateAction(valParams);
if (vStat && vStat.code !== 200) {
if (vStat?.code !== 200) {
return vStat;
}

respPayload = 'Started deleting content';
logger.info(respPayload);
const siteFgRootPath = appConfig.getSiteFgRootPath();
const batchManager = new BatchManager({
key: DELETE_ACTION,
instanceKey: getInstanceKey({ fgRootFolder: siteFgRootPath }),
batchConfig: appConfig.getDeleteBatchConfig()
});

await batchManager.init();
// Clean up files before starting
await batchManager.cleanupFiles();

await fgStatus.updateStatusToStateLib({
status: FgStatus.PROJECT_STATUS.IN_PROGRESS,
statusMessage: respPayload
});

const sharepoint = new Sharepoint(appConfig);
const deleteStatus = await sharepoint.deleteFloodgateDir();
respPayload = deleteStatus === false ?
'Error occurred when deleting content. Check project excel sheet for additional information.' :
'Delete action was completed';
const helixUtils = new HelixUtils(appConfig);
const filesToUnpublish = await helixUtils.getFilesToUnpublish(fgColor);
logger.info(`List of files to unpublish: ${filesToUnpublish?.length}`);

await fgStatus.updateStatusToStateLib({
status: deleteStatus === false ? FgStatus.PROJECT_STATUS.COMPLETED_WITH_ERROR : FgStatus.PROJECT_STATUS.COMPLETED,
statusMessage: respPayload
});
if (!filesToUnpublish) {
throw new Error('Unable to get items to unpublish! Please retry after some time.');
} else if (filesToUnpublish.length > 0) {
await filesToUnpublish.reduce(async (acc, curr) => {
await acc;
await batchManager.addFile(curr);
}, Promise.resolve());

const { startTime: startDelete, endTime: endDelete } = fgStatus.getStartEndTime();
const excelValues = [['DELETE', toUTCStr(startDelete), toUTCStr(endDelete), respPayload]];
logger.info('Finalize instance');
await batchManager.finalizeInstance(appConfig.getPassthruParams());
} else {
await fgDeleteActionHelper.completeProcess(projectExcelPath, batchManager, [], fgStatus, sharepoint);
}

await sharepoint.updateExcelTable(projectExcelPath, 'DELETE_STATUS', excelValues);
logger.info('Project excel file updated with delete status.');
logger.info('Batching completed');
} catch (err) {
await fgDeleteActionHelper.completeProcess();
await fgStatus.updateStatusToStateLib({
status: FgStatus.PROJECT_STATUS.COMPLETED_WITH_ERROR,
statusMessage: err.message
Expand Down
2 changes: 1 addition & 1 deletion actions/delete/delete.js
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ async function main(args) {
statusMessage: 'Triggering delete action'
});
return ow.actions.invoke({
name: 'milo-fg/delete-worker',
name: 'milo-fg/delete-create-batch',
blocking: false, // this is the flag that instructs to execute the worker asynchronous
result: false,
params: args
Expand Down
132 changes: 132 additions & 0 deletions actions/delete/triggerNTrack.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/* eslint-disable no-await-in-loop */
/* ************************************************************************
* ADOBE CONFIDENTIAL
* ___________________
*
* Copyright 2023 Adobe
* All Rights Reserved.
*
* NOTICE: All information contained herein is, and remains
* the property of Adobe and its suppliers, if any. The intellectual
* and technical concepts contained herein are proprietary to Adobe
* and its suppliers and are protected by all applicable intellectual
* property laws, including trade secret and copyright laws.
* Dissemination of this information or reproduction of this material
* is strictly forbidden unless prior written permission is obtained
* from Adobe.
************************************************************************* */
const openwhisk = require('openwhisk');
const Sharepoint = require('../sharepoint');
const {
successResponse, getAioLogger, DELETE_ACTION,
} = require('../utils');
const AppConfig = require('../appConfig');
const FgStatus = require('../fgStatus');
const BatchManager = require('../batchManager');
const FgAction = require('../fgAction');
const FgDeleteActionHelper = require('../fgDeleteActionHelper');

const logger = getAioLogger();

async function main(params) {
let respPayload;

const valParams = {
statParams: ['fgRootFolder'],
actParams: ['adminPageUri', 'projectExcelPath'],
checkUser: false,
checkStatus: false,
checkActivation: false,
};

const ow = openwhisk();
let appConfig = new AppConfig(params);
const fgDeleteActionHelper = new FgDeleteActionHelper();
const batchManager = new BatchManager({
key: DELETE_ACTION,
batchConfig: appConfig.getDeleteBatchConfig(),
});

await batchManager.init();

if (await batchManager.isInstanceRunning()) {
return successResponse('Skipping, Instance is running!');
}

// Read instance_info.json
const instanceContent = await batchManager.getInstanceData();
if (!instanceContent?.dtls) {
return successResponse('None to run!');
}

const { batchesInfo } = instanceContent.dtls;

// Initialize action
appConfig = new AppConfig({ ...params, ...instanceContent.dtls });
const fgAction = new FgAction(DELETE_ACTION, appConfig);
fgAction.init({ ow, skipUserDetails: true });
const { fgStatus } = fgAction.getActionParams();

try {
await batchManager.markInstanceRunning();
const vStat = await fgAction.validateAction(valParams);
if (vStat && vStat.code !== 200) {
return vStat;
}

const { payload } = appConfig.getConfig();

// Find the batch to be triggered
const total = batchesInfo.length;
const nextBatch = batchesInfo.find((b) => !b.done);
const batchNumber = nextBatch?.batchNumber;

if (!batchNumber) {
await fgStatus.updateStatusToStateLib({ status: FgStatus.PROJECT_STATUS.COMPLETED });
return successResponse('None to be processed!');
}

respPayload = `Unpublishing batch ${batchNumber} / ${total}`;
await fgStatus.updateStatusToStateLib({
status: FgStatus.PROJECT_STATUS.IN_PROGRESS,
statusMessage: respPayload,
});

logger.info(respPayload);
await fgDeleteActionHelper.processBatch(appConfig, batchManager, nextBatch);
await batchManager.writeToInstanceFile(instanceContent);

// Complete the process when all batches are completed
const hasPendingBatches = batchesInfo.find((b) => !b.done);
logger.info(`Has pending batches: ${hasPendingBatches}`);

if (!hasPendingBatches) {
const sharepoint = new Sharepoint(appConfig);
await fgDeleteActionHelper.completeProcess(payload.projectExcelPath, batchManager, batchesInfo, fgStatus, sharepoint);
respPayload = 'Delete process completed.';
}

logger.info(respPayload);
respPayload = 'Delete trigger and track completed.';
} catch (err) {
logger.error(err);
respPayload = err;

// In case of error, log status with end time
try {
await fgStatus.updateStatusToStateLib({
status: FgStatus.PROJECT_STATUS.COMPLETED_WITH_ERROR,
statusMessage: err.message,
});
} catch (err2) {
logger.info('Error while updating failed status');
}
}

await batchManager.markInstancePaused();
return {
body: respPayload,
};
}

exports.main = main;
Loading

0 comments on commit 3fb4bed

Please sign in to comment.