Merge pull request #1559 from akto-api-security/hotfix/fix_clean_inve…

…ntory_job Fixing delete filter case and adding collectionwise summary
akto-api-security · Sep 27, 2024 · 1601827 · 1601827
2 parents b3fc8a9 + 7950af3
commit 1601827
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 5 deletions.
diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java
@@ -214,7 +214,7 @@ public static FILTER_TYPE isValidResponseParam(HttpResponseParams responseParam,
                 }
             } catch (Exception e) {
                 loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString()));
-                filterType = FILTER_TYPE.UNCHANGED;
+                filterType = FILTER_TYPE.ERROR;
             }
         }
         return filterType;
@@ -231,7 +231,7 @@ public static Pair<HttpResponseParams,FILTER_TYPE> applyAdvancedFilters(HttpResp
                 return new Pair<HttpResponseParams,FilterConfig.FILTER_TYPE>(responseParams, filterType);
             }
         }
-        return new Pair<HttpResponseParams,FilterConfig.FILTER_TYPE>(responseParams, FILTER_TYPE.UNCHANGED);
+        return new Pair<HttpResponseParams,FilterConfig.FILTER_TYPE>(responseParams, FILTER_TYPE.ERROR);
     }
 
     public void syncFunction(List<HttpResponseParams> responseParams, boolean syncImmediately, boolean fetchAllSTI, AccountSettings accountSettings)  {
@@ -577,7 +577,7 @@ public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams
 
             Pair<HttpResponseParams,FILTER_TYPE> temp = applyAdvancedFilters(httpResponseParam, executorNodesMap, apiCatalogSync.advancedFilterMap);
             HttpResponseParams param = temp.getFirst();
-            if(param == null){
+            if(param == null || temp.getSecond().equals(FILTER_TYPE.UNCHANGED)){
                 continue;
             }else{
                 httpResponseParam = param;

diff --git a/apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java b/apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java
@@ -101,7 +101,7 @@ public String syncTrafficFromFilters(){
             }
 
             List<ApiCollection> apiCollections = ApiCollectionsDao.instance.findAll(
-                Filters.empty(), Projections.include(ApiCollection.HOST_NAME));
+                Filters.empty(), Projections.include(ApiCollection.HOST_NAME, ApiCollection.NAME));
             YamlTemplate yamlTemplate = new YamlTemplate(filterConfig.getId(), Context.now(), getSUser().getLogin(), Context.now(), this.yamlContent, null);
 
             CleanInventory.cleanFilteredSampleDataFromAdvancedFilters(apiCollections,Arrays.asList(yamlTemplate),new ArrayList<>() , "", false, true);

diff --git a/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java b/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java
@@ -6,6 +6,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -134,6 +135,7 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List<ApiCollection
         int skip = 0;
         int limit = 100;
         Bson sort = Sorts.ascending("_id.apiCollectionId", "_id.url", "_id.method");
+        Map<Integer,Integer> collectionWiseDeletionCountMap = new HashMap<>();
 
         Map<String,FilterConfig> filterMap = FilterYamlTemplateDao.instance.fetchFilterConfig(false, yamlTemplates, true);
         Pattern pattern = createRegexPatternFromList(redundantUrlList);
@@ -173,17 +175,30 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List<ApiCollection
                                 FILTER_TYPE filterType = temp.getSecond();
 
                                 if(param != null){
+                                    // comes when Filter_Block is not valid {Remaining => Unchanged, Modified, Allowed}
                                     if(filterType.equals(FILTER_TYPE.MODIFIED)){
+                                        // filter passed and modified
                                         movingApi = true;
+                                        break;
                                     }else if(filterType.equals(FILTER_TYPE.ALLOWED)){
+                                        // filter passed and not modified
                                         isAllowedFromTemplate = true;
+                                    }else if(filterMap.size() == 1){
+                                        // filter failed and id was default_delete
+                                        String key = filterMap.entrySet().iterator().next().getKey();
+                                        if(key.equals("DEFAULT_BLOCK_FILTER")){
+                                            isAllowedFromTemplate = true;
+                                        }
                                     }
+                                }else{
+                                    break;
                                 }
                             }
                         }
                     }
 
                     if(movingApi){
+                        // any 1 of the sample is modifiable, we print this block
                         toMove.add(sampleData.getId());
                         if(saveLogsToDB){
                             loggerMaker.infoAndAddToDb("Filter passed, modify sample data of API: " + sampleData.getId(), LogDb.DASHBOARD);
@@ -194,6 +209,9 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List<ApiCollection
 
                     else if (isRedundant || !isAllowedFromTemplate) {                                
                         // writer.write(sampleData.toString());
+                        // if api falls under redundant url and if block filter is passed or none of the filter from any of the filters is passed, we print this block
+                        int initialCount = collectionWiseDeletionCountMap.getOrDefault(sampleData.getId().getApiCollectionId(), 0);
+                        collectionWiseDeletionCountMap.put(sampleData.getId().getApiCollectionId(),initialCount + 1);
                         toBeDeleted.add(sampleData.getId());  
                         if(saveLogsToDB){
                             loggerMaker.infoAndAddToDb(
@@ -203,6 +221,7 @@ else if (isRedundant || !isAllowedFromTemplate) {
                             logger.info("[BadApisRemover] " + isNetsparkerPresent + " Deleting bad API from template: " + sampleData.getId(), LogDb.DASHBOARD);
                         }           
                     } else {
+                        // other cases like: => filter from advanced filter is passed || filter from block filter fails
                         if(saveLogsToDB){
                             loggerMaker.infoAndAddToDb(
                                 "Filter did not pass, keeping api found from filter: " + sampleData.getId(), LogDb.DASHBOARD
@@ -225,6 +244,16 @@ else if (isRedundant || !isAllowedFromTemplate) {
 
         } while (!sampleDataList.isEmpty());
 
+        for(Map.Entry<Integer,Integer> iterator: collectionWiseDeletionCountMap.entrySet()){
+            int collId = iterator.getKey();
+            int deletionCount = iterator.getValue();
+            String name = apiCollectionMap.get(collId).getDisplayName();
+
+            if(saveLogsToDB){
+                loggerMaker.infoAndAddToDb("Total apis deleted from collection: " + name + " are: " + deletionCount, LogDb.DASHBOARD);
+            }
+        }
+
         // writer.flush();
         // writer.close();
     }

diff --git a/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java b/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java
@@ -27,7 +27,7 @@ public class FilterConfig {
     public static final String DEFAULT_BLOCK_FILTER = "DEFAULT_BLOCK_FILTER";
 
     public enum FILTER_TYPE{
-        BLOCKED , ALLOWED, MODIFIED, UNCHANGED
+        BLOCKED , ALLOWED, MODIFIED, UNCHANGED, ERROR
     }
 
     private ExecutorConfigParserResult executor;