Merge pull request #1710 from akto-api-security/hotfix/add_max_retrie…

…s_for_Tests Adding max retries for failed tests
akto-api-security · Nov 13, 2024 · 106e31b · 106e31b
2 parents e8b939e + ca5ebc7
commit 106e31b
Showing 1 changed file with 44 additions and 16 deletions.
diff --git a/apps/testing/src/main/java/com/akto/testing/Main.java b/apps/testing/src/main/java/com/akto/testing/Main.java
@@ -136,6 +136,7 @@ public static Set<Integer> extractApiCollectionIds(List<ApiInfo.ApiInfoKey> apiI
     }
     private static final int LAST_TEST_RUN_EXECUTION_DELTA = 5 * 60;
     private static final int DEFAULT_DELTA_IGNORE_TIME = 2*60*60;
+    private static final int MAX_RETRIES_FOR_FAILED_SUMMARIES = 3;
 
     private static TestingRun findPendingTestingRun(int userDeltaTime) {
         int deltaPeriod = userDeltaTime == 0 ? DEFAULT_DELTA_IGNORE_TIME : userDeltaTime;
@@ -344,7 +345,7 @@ public void run() {
                         }
                     }
                     setTestingRunConfig(testingRun, trrs);
-
+                    boolean maxRetriesReached = false;
                     if (isSummaryRunning || isTestingRunRunning) {
                         loggerMaker.infoAndAddToDb("TRRS or TR is in running state, checking if it should run it or not");
                         TestingRunResultSummary testingRunResultSummary;
@@ -353,7 +354,7 @@ public void run() {
                         } else {
                             Map<ObjectId, TestingRunResultSummary> objectIdTestingRunResultSummaryMap = TestingRunResultSummariesDao.instance.fetchLatestTestingRunResultSummaries(Collections.singletonList(testingRun.getId()));
                             testingRunResultSummary = objectIdTestingRunResultSummaryMap.get(testingRun.getId());
-                        }
+                        }                   
 
                         if (testingRunResultSummary != null) {
                             List<TestingRunResult> testingRunResults = TestingRunResultDao.instance.fetchLatestTestingRunResult(Filters.eq(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, testingRunResultSummary.getId()), 1);
@@ -366,12 +367,31 @@ public void run() {
                                 } else {
                                     loggerMaker.infoAndAddToDb("Test run was executed long ago, TRR_ID:"
                                             + testingRunResult.getHexId() + ", TRRS_ID:" + testingRunResultSummary.getHexId() + " TR_ID:" + testingRun.getHexId(), LogDb.TESTING);
+
+                                    int maxRunTime = testingRun.getTestRunTime() <= 0 ? 30*60 : testingRun.getTestRunTime(); 
+                                    Bson filterQ = Filters.and(
+                                        Filters.gte(TestingRunResultSummary.START_TIMESTAMP, (Context.now() - ((MAX_RETRIES_FOR_FAILED_SUMMARIES + 1) * maxRunTime))),
+                                        Filters.eq(TestingRunResultSummary.TESTING_RUN_ID, testingRun.getId()),
+                                        Filters.eq(TestingRunResultSummary.STATE, State.FAILED)
+                                    );
+
+                                    int countFailedSummaries = (int) TestingRunResultSummariesDao.instance.count(filterQ);
+                                    Bson updateForSummary = Updates.set(TestingRunResultSummary.STATE, State.FAILED);
+                                    if(countFailedSummaries >= (MAX_RETRIES_FOR_FAILED_SUMMARIES - 1)){
+                                        updateForSummary = Updates.combine(
+                                            Updates.set(TestingRunResultSummary.STATE, State.COMPLETED),
+                                            Updates.set(TestingRunResultSummary.END_TIMESTAMP, Context.now())
+                                        );
+                                        loggerMaker.infoAndAddToDb("Max retries level reached for TRR_ID: " + testingRun.getHexId(), LogDb.TESTING);
+                                        maxRetriesReached = true;
+                                    }
+
                                     TestingRunResultSummary summary = TestingRunResultSummariesDao.instance.updateOneNoUpsert(
                                             Filters.and(
                                                     Filters.eq(TestingRunResultSummary.ID, testingRunResultSummary.getId()),
                                                     Filters.eq(TestingRunResultSummary.STATE, State.RUNNING)
                                             ),
-                                            Updates.set(TestingRunResultSummary.STATE, State.FAILED)
+                                            updateForSummary
                                     );
                                     if (summary == null) {
                                         loggerMaker.infoAndAddToDb("Skipping because some other thread picked it up, TRRS_ID:" + testingRunResultSummary.getHexId() + " TR_ID:" + testingRun.getHexId(), LogDb.TESTING);
@@ -397,17 +417,22 @@ public void run() {
                             }
 
                             // insert new summary based on old summary
-                            if (summaryId != null) {
-                                trrs.setId(new ObjectId());
-                                trrs.setStartTimestamp(start);
-                                trrs.setState(State.RUNNING);
-                                trrs.setTestResultsCount(0);
-                                trrs.setCountIssues(emptyCountIssuesMap);
-                                TestingRunResultSummariesDao.instance.insertOne(trrs);
-                                summaryId = trrs.getId();
-                            } else {
-                                trrs = createTRRSummaryIfAbsent(testingRun, start);
-                                summaryId = trrs.getId();
+                            // add max retries here and then mark last summary as completed when results > 0
+                            if(maxRetriesReached){
+                                loggerMaker.infoAndAddToDb("Exiting out as maxRetries have been reached for testingRun: " + testingRun.getHexId(), LogDb.TESTING);
+                            }else{
+                                if (summaryId != null) {
+                                    trrs.setId(new ObjectId());
+                                    trrs.setStartTimestamp(start);
+                                    trrs.setState(State.RUNNING);
+                                    trrs.setTestResultsCount(0);
+                                    trrs.setCountIssues(emptyCountIssuesMap);
+                                    TestingRunResultSummariesDao.instance.insertOne(trrs);
+                                    summaryId = trrs.getId();
+                                } else {
+                                    trrs = createTRRSummaryIfAbsent(testingRun, start);
+                                    summaryId = trrs.getId();
+                                }
                             }
                         } else {
                             loggerMaker.infoAndAddToDb("No summary found. Let's run it as usual");
@@ -428,8 +453,11 @@ public void run() {
                         }
                     }
                     RequiredConfigs.initiate();
-                    testExecutor.init(testingRun, summaryId, syncLimit);
-                    raiseMixpanelEvent(summaryId, testingRun, accountId);
+                    if(!maxRetriesReached){
+                        testExecutor.init(testingRun, summaryId, syncLimit);
+                        raiseMixpanelEvent(summaryId, testingRun, accountId);
+                    }
+
             } catch (Exception e) {
                     loggerMaker.errorAndAddToDb(e, "Error in init " + e);
                 }