Skip to content

Commit

Permalink
Merge pull request #1507 from akto-api-security/fix/add_logs_per_account
Browse files Browse the repository at this point in the history
Allowing logs for saas account
  • Loading branch information
avneesh-akto authored Sep 18, 2024
2 parents 9181112 + 5e522d3 commit d9ea1b1
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -631,31 +631,36 @@ public void run() {
AccountTask.instance.executeTask(new Consumer<Account>() {
@Override
public void accept(Account account) {
AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter());
boolean detectRedundantUrls = accountSettings.getAllowFilterLogs();
String shouldFilterApisFromYaml = System.getenv("DETECT_REDUNDANT_APIS_RETRO");
String shouldFilterOptionsAndHTMLApis = System.getenv("DETECT_OPTION_APIS_RETRO");
if(shouldFilterApisFromYaml == null && shouldFilterOptionsAndHTMLApis == null){

String shouldDelete = System.getenv("DELETE_REDUNDANT_APIS");
boolean shouldDeleteApis = accountSettings.getAllowDeletionOfUrls() || (shouldDelete != null && shouldDelete.equalsIgnoreCase("true"));

if(!detectRedundantUrls && shouldFilterApisFromYaml == null && shouldFilterOptionsAndHTMLApis == null){
return;
}
List<ApiCollection> apiCollections = ApiCollectionsDao.instance.findAll(Filters.empty(),
Projections.include(Constants.ID, ApiCollection.NAME, ApiCollection.HOST_NAME));

String filePath = "./samples_"+account.getId()+".txt";

if(shouldFilterApisFromYaml != null && shouldFilterApisFromYaml.equalsIgnoreCase("true")){
if((detectRedundantUrls || shouldFilterApisFromYaml != null && shouldFilterApisFromYaml.equalsIgnoreCase("true"))){
List<YamlTemplate> yamlTemplates = AdvancedTrafficFiltersDao.instance.findAll(
Filters.ne(YamlTemplate.INACTIVE, true)
);
AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter());
List<String> redundantUrlList = accountSettings.getAllowRedundantEndpointsList();
try {
CleanInventory.cleanFilteredSampleDataFromAdvancedFilters(apiCollections , yamlTemplates, redundantUrlList,filePath, false);
CleanInventory.cleanFilteredSampleDataFromAdvancedFilters(apiCollections , yamlTemplates, redundantUrlList,filePath, shouldDeleteApis);
} catch (Exception e) {
e.printStackTrace();
}
}

if(shouldFilterOptionsAndHTMLApis != null && shouldFilterOptionsAndHTMLApis.equalsIgnoreCase("true")){
CleanInventory.removeUnnecessaryEndpoints(apiCollections);
CleanInventory.removeUnnecessaryEndpoints(apiCollections, shouldDeleteApis);
}
}
}, "clean-inventory-job");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -126,7 +125,7 @@ private static void cleanInventoryJob() {

}

public static void cleanFilteredSampleDataFromAdvancedFilters(List<ApiCollection> apiCollections, List<YamlTemplate> yamlTemplates, List<String> redundantUrlList, String filePath, boolean shouldModifyRequest) throws IOException{
public static void cleanFilteredSampleDataFromAdvancedFilters(List<ApiCollection> apiCollections, List<YamlTemplate> yamlTemplates, List<String> redundantUrlList, String filePath, boolean shouldDeleteRequest) throws IOException{

Map<Integer, ApiCollection> apiCollectionMap = apiCollections.stream().collect(Collectors.toMap(ApiCollection::getId, Function.identity()));
// BufferedWriter writer = new BufferedWriter(new FileWriter(new File(filePath)));
Expand Down Expand Up @@ -158,26 +157,27 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List<ApiCollection
}


boolean allMatchDefault = false;
boolean isRedundant = false;
boolean isAllowedFromTemplate = false;
boolean isNetsparkerPresent = false;
boolean movingApi = false;
for (String sample : samples) {
HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample);
isNetsparkerPresent |= sample.toLowerCase().contains("netsparker");
if(httpResponseParams != null){
allMatchDefault = HttpCallParser.isRedundantEndpoint(httpResponseParams.getRequestParams().getURL(), pattern);
if(!allMatchDefault){
isRedundant = HttpCallParser.isRedundantEndpoint(httpResponseParams.getRequestParams().getURL(), pattern);
if(!isRedundant){
Map<String, List<ExecutorNode>> executorNodesMap = ParseAndExecute.createExecutorNodeMap(filterMap);
Pair<HttpResponseParams,FILTER_TYPE> temp = HttpCallParser.applyAdvancedFilters(httpResponseParams, executorNodesMap, filterMap);
HttpResponseParams param = temp.getFirst();
FILTER_TYPE filterType = temp.getSecond();

if(param != null){
allMatchDefault = false;
if(temp.getSecond().equals(FILTER_TYPE.MODIFIED)){
if(filterType.equals(FILTER_TYPE.MODIFIED)){
movingApi = true;
}else if(filterType.equals(FILTER_TYPE.ALLOWED)){
isAllowedFromTemplate = true;
}
}else{
allMatchDefault = true;
}
}
}
Expand All @@ -188,32 +188,31 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List<ApiCollection
logger.info("[BadApisUpdater] Updating bad from template API: " + sampleData.getId(), LogDb.DASHBOARD);
}

else if (allMatchDefault) {
else if (isRedundant || !isAllowedFromTemplate) {
// writer.write(sampleData.toString());
toBeDeleted.add(sampleData.getId());
logger.info("[BadApisRemover] " + isNetsparkerPresent + " Deleting bad API from template: " + sampleData.getId(), LogDb.DASHBOARD);
} else {
logger.info("[BadApisRemover] " + isNetsparkerPresent + " Keeping bad API from template: " + sampleData.getId(), LogDb.DASHBOARD);
logger.info("[BadApisRemover] " + isNetsparkerPresent + " Keeping API from template: " + sampleData.getId(), LogDb.DASHBOARD);
}
} catch (Exception e) {
loggerMaker.errorAndAddToDb("[BadApisRemover] Couldn't delete an api for default payload: " + sampleData.getId() + e.getMessage(), LogDb.DASHBOARD);
}
}

String shouldDelete = System.getenv("DELETE_REDUNDANT_APIS");
if ( shouldDelete != null && shouldDelete.equalsIgnoreCase("true")) {
if (shouldDeleteRequest) {
logger.info("starting deletion of apis");
deleteApis(toBeDeleted);
}

String shouldMove = System.getenv("MOVE_REDUNDANT_APIS");
// String shouldMove = System.getenv("MOVE_REDUNDANT_APIS");

} while (!sampleDataList.isEmpty());

// writer.flush();
// writer.close();
}

public static void removeUnnecessaryEndpoints(List<ApiCollection> apiCollections){
public static void removeUnnecessaryEndpoints(List<ApiCollection> apiCollections, boolean shouldDeleteRequest){
try {
for (ApiCollection apiCollection: apiCollections) {
List<Key> toBeDeleted = new ArrayList<>();
Expand Down Expand Up @@ -270,8 +269,9 @@ public static void removeUnnecessaryEndpoints(List<ApiCollection> apiCollections
}
}

String shouldDelete = System.getenv("DELETE_REDUNDANT_APIS");
if ( shouldDelete != null && shouldDelete.equalsIgnoreCase("true")) {

if (shouldDeleteRequest) {
logger.info("starting deletion of apis");
deleteApis(toBeDeleted);
}
}
Expand Down
22 changes: 22 additions & 0 deletions libs/dao/src/main/java/com/akto/dto/AccountSettings.java
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,12 @@ public class AccountSettings {
public static final String ALLOW_SENDING_EVENTS_TO_INTERCOM = "allowSendingEventsToIntercom";
private boolean allowSendingEventsToIntercom;

public static final String ALLOW_FILTER_LOGS = "allowFilterLogs";
private boolean allowFilterLogs;

public static final String ALLOW_DELETION_OF_REDUNDANT_URLS = "allowDeletionOfUrls";
private boolean allowDeletionOfUrls;

private static final List<String> defaultCidrRangesList = Arrays.asList("10.0.0.0/8", "172.16.0.0/12",
"192.168.0.0/16", "127.0.0.0/8", "169.254.0.0/16", "224.0.0.0/4", "192.0.2.0/24", "198.51.100.0/24",
"203.0.113.0/24", "255.255.255.255/32", "100.64.0.0/10", "192.88.99.0/24", "240.0.0.0/4");
Expand Down Expand Up @@ -401,4 +407,20 @@ public boolean getAllowSendingEventsToIntercom() {
public void setAllowSendingEventsToIntercom(boolean allowSendingEventsToIntercom) {
this.allowSendingEventsToIntercom = allowSendingEventsToIntercom;
}

public boolean getAllowFilterLogs() {
return allowFilterLogs;
}

public void setAllowFilterLogs(boolean allowFilterLogs) {
this.allowFilterLogs = allowFilterLogs;
}

public boolean getAllowDeletionOfUrls() {
return allowDeletionOfUrls;
}

public void setAllowDeletionOfUrls(boolean allowDeletionOfUrls) {
this.allowDeletionOfUrls = allowDeletionOfUrls;
}
}

0 comments on commit d9ea1b1

Please sign in to comment.