Skip to content

Commit

Permalink
Merge pull request #1719 from akto-api-security/hotfix/delete_options…
Browse files Browse the repository at this point in the history 
…_urls

Deleting options apis by default
  • Loading branch information
@avneesh-akto
avneesh-akto authored Nov 13, 2024
2 parents 5a421e9 + 31bd741 commit 6c1ac0d
Show file tree
Hide file tree
Showing 6 changed files with 87 additions and 14 deletions.
15 changes: 12 additions & 3 deletions apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,11 @@ public void syncFunction(List<HttpResponseParams> responseParams, boolean syncIm
redundantList = accountSettings.getAllowRedundantEndpointsList();
}
Pattern regexPattern = Utils.createRegexPatternFromList(redundantList);
filteredResponseParams = filterHttpResponseParams(filteredResponseParams, redundantList, regexPattern);
boolean shouldIgnoreOptionsApi = true;
if(accountSettings != null){
shouldIgnoreOptionsApi = !accountSettings.getAllowOptionsAPIs();
}
filteredResponseParams = filterHttpResponseParams(filteredResponseParams, redundantList, regexPattern, shouldIgnoreOptionsApi);

boolean makeApisCaseInsensitive = false;
if(accountSettings != null){
Expand Down Expand Up @@ -516,7 +520,7 @@ private boolean isBlankResponseBodyForGET(String method, String contentType, Str
return res;
}

public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams> httpResponseParamsList, List<String> redundantUrlsList, Pattern pattern) {
public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams> httpResponseParamsList, List<String> redundantUrlsList, Pattern pattern, Boolean shouldIgnoreOptionsApi) {
List<HttpResponseParams> filteredResponseParams = new ArrayList<>();
int originalSize = httpResponseParamsList.size();

Expand All @@ -539,6 +543,11 @@ public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams

String ignoreAktoFlag = getHeaderValue(httpResponseParam.getRequestParams().getHeaders(),Constants.AKTO_IGNORE_FLAG);
if (ignoreAktoFlag != null) continue;

String method = httpResponseParam.getRequestParams().getMethod();
if(shouldIgnoreOptionsApi && method != null && method.equalsIgnoreCase("OPTIONS")){
continue;
}

// check for garbage points here
if(redundantUrlsList != null && !redundantUrlsList.isEmpty()){
Expand All @@ -557,7 +566,7 @@ public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams
try {
List<String> responseContentTypeList = (List<String>) httpResponseParam.getHeaders().getOrDefault("content-type", new ArrayList<>());
String allContentTypes = responseContentTypeList.toString();
String method = httpResponseParam.getRequestParams().getMethod();

String responseBody = httpResponseParam.getPayload();
boolean ignore = false;
for (String extension : redundantUrlsList) {
Expand Down
20 changes: 10 additions & 10 deletions apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ public void testInitialiseFilters() throws InterruptedException {
@Test
public void testFilterHttpResponseParamsEmpty() {
HttpCallParser httpCallParser = new HttpCallParser("",0,0,0, true);
List<HttpResponseParams> ss = httpCallParser.filterHttpResponseParams(new ArrayList<>(), null, null);
List<HttpResponseParams> ss = httpCallParser.filterHttpResponseParams(new ArrayList<>(), null, null, true);
assertEquals(ss.size(),0);
}

Expand All @@ -283,7 +283,7 @@ public void testFilterHttpResponseParamsIpHost() {
h2.requestParams.setApiCollectionId(1000);
h2.setSource(Source.MIRRORING);

List<HttpResponseParams> ss = httpCallParser.filterHttpResponseParams(Arrays.asList(h1, h2), null, null);
List<HttpResponseParams> ss = httpCallParser.filterHttpResponseParams(Arrays.asList(h1, h2), null, null, true);
assertEquals(ss.size(),2);
assertEquals(h1.requestParams.getApiCollectionId(), 1000);
assertTrue(h2.requestParams.getApiCollectionId() != 1000);
Expand All @@ -307,7 +307,7 @@ public void testFilterHttpResponseParamsWithoutHost() {
h1.statusCode = 200;
h1.requestParams.setApiCollectionId(vxlanId1);

List<HttpResponseParams> filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null, null);
List<HttpResponseParams> filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null, null, true);

Assertions.assertEquals(filterHttpResponseParamsList.size(),1);
Assertions.assertEquals(filterHttpResponseParamsList.get(0).requestParams.getApiCollectionId(),vxlanId1);
Expand All @@ -322,7 +322,7 @@ public void testFilterHttpResponseParamsWithoutHost() {
h2.statusCode = 200;
h2.requestParams.setApiCollectionId(vxlanId2);

filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h2),null, null);
filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h2),null, null, true);

Assertions.assertEquals(filterHttpResponseParamsList.size(),1);
Assertions.assertEquals(filterHttpResponseParamsList.get(0).requestParams.getApiCollectionId(),vxlanId2);
Expand All @@ -337,7 +337,7 @@ public void testFilterHttpResponseParamsWithoutHost() {
h3.statusCode = 400;
h3.requestParams.setApiCollectionId(vxlanId2);

filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h3),null, null);
filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h3),null, null, true);

Assertions.assertEquals(filterHttpResponseParamsList.size(),0);
ApiCollection apiCollection3 = ApiCollectionsDao.instance.findOne("_id", vxlanId3);
Expand Down Expand Up @@ -369,7 +369,7 @@ public void testFilterResponseParamsWithHost() {
h1.statusCode = 200;
h1.setSource(Source.MIRRORING);

httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null, null);
httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null, null, true);

List<ApiCollection> apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject());
Assertions.assertEquals(apiCollections.size(),2);
Expand All @@ -395,7 +395,7 @@ public void testFilterResponseParamsWithHost() {
h2.statusCode = 200;
h2.setSource(Source.MIRRORING);

httpCallParser.filterHttpResponseParams(Collections.singletonList(h2),null, null);
httpCallParser.filterHttpResponseParams(Collections.singletonList(h2),null, null, true);

apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject());
Assertions.assertEquals(apiCollections.size(),3);
Expand All @@ -419,7 +419,7 @@ public void testFilterResponseParamsWithHost() {
h3.statusCode = 200;
h3.setSource(Source.MIRRORING);

httpCallParser.filterHttpResponseParams(Collections.singletonList(h3),null, null);
httpCallParser.filterHttpResponseParams(Collections.singletonList(h3),null, null, true);

apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject());
Assertions.assertEquals(apiCollections.size(),4);
Expand Down Expand Up @@ -452,7 +452,7 @@ public void testFilterResponseParamsWithHost() {
);
httpCallParser.getHostNameToIdMap().put("hostRandom 1234", dupId);

httpCallParser.filterHttpResponseParams(Collections.singletonList(h4),null, null);
httpCallParser.filterHttpResponseParams(Collections.singletonList(h4),null, null, true);

apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject());
Assertions.assertEquals(apiCollections.size(),6);
Expand All @@ -476,7 +476,7 @@ public void testCollisionHostNameCollection() {
h1.statusCode = 200;

HttpCallParser httpCallParser = new HttpCallParser("",0,0,0, true);
httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null, null);
httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null, null, true);

List<ApiCollection> apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject());
Assertions.assertEquals(apiCollections.size(), 1);
Expand Down
13 changes: 13 additions & 0 deletions apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2850,6 +2850,18 @@ private static void moveAzureSamlConfig(BackwardCompatibility backwardCompatibil
}
}

private static void deleteOptionsAPIs(BackwardCompatibility backwardCompatibility){
if(backwardCompatibility.getDeleteOptionsAPIs() == 0){
List<ApiCollection> apiCollections = ApiCollectionsDao.instance.findAll(Filters.nin(Constants.ID,UsageMetricCalculator.getDeactivated()),
Projections.include(Constants.ID, ApiCollection.NAME, ApiCollection.HOST_NAME));
CleanInventory.deleteOptionsAPIs(apiCollections);
BackwardCompatibilityDao.instance.updateOne(
Filters.eq("_id", backwardCompatibility.getId()),
Updates.set(BackwardCompatibility.DELETE_OPTIONS_API, Context.now())
);
}
}

public static void setBackwardCompatibilities(BackwardCompatibility backwardCompatibility){
if (DashboardMode.isMetered()) {
initializeOrganizationAccountBelongsTo(backwardCompatibility);
Expand All @@ -2871,6 +2883,7 @@ public static void setBackwardCompatibilities(BackwardCompatibility backwardComp
createRiskScoreGroups(backwardCompatibility);
setApiCollectionAutomatedField(backwardCompatibility);
createAutomatedAPIGroups(backwardCompatibility);
deleteOptionsAPIs(backwardCompatibility);
deleteAccessListFromApiToken(backwardCompatibility);
deleteNullSubCategoryIssues(backwardCompatibility);
enableNewMerging(backwardCompatibility);
Expand Down
28 changes: 28 additions & 0 deletions apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,4 +331,32 @@ public static void removeUnnecessaryEndpoints(List<ApiCollection> apiCollections
}
}

public static void deleteOptionsAPIs(List<ApiCollection> apiCollections){
for (ApiCollection apiCollection: apiCollections) {
List<Key> toBeDeleted = new ArrayList<>();
if (apiCollection.getHostName() == null) {
continue;
}
List<BasicDBObject> endpoints = com.akto.action.observe.Utils.fetchEndpointsInCollectionUsingHost(apiCollection.getId(), 0);

if (endpoints == null || endpoints.isEmpty()) {
continue;
}
for (BasicDBObject singleTypeInfo: endpoints) {
singleTypeInfo = (BasicDBObject) (singleTypeInfo.getOrDefault("_id", new BasicDBObject()));
int apiCollectionId = singleTypeInfo.getInt("apiCollectionId");
String url = singleTypeInfo.getString("url");
String method = singleTypeInfo.getString("method");

Key key = new Key(apiCollectionId, url, Method.fromString(method), -1, 0, 0);

if (method.equalsIgnoreCase("options")) {
toBeDeleted.add(key);
continue;
}
}
deleteApis(toBeDeleted);
}
}

}
11 changes: 11 additions & 0 deletions libs/dao/src/main/java/com/akto/dto/AccountSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ public class AccountSettings {
public static final String ALLOW_DELETION_OF_REDUNDANT_URLS = "allowDeletionOfUrls";
private boolean allowDeletionOfUrls;

public static final String ALLOW_OPTIONS_API = "allowOptionsAPIs";
private boolean allowOptionsAPIs;

private static final List<String> defaultCidrRangesList = Arrays.asList("10.0.0.0/8", "172.16.0.0/12",
"192.168.0.0/16", "127.0.0.0/8", "169.254.0.0/16", "224.0.0.0/4", "192.0.2.0/24", "198.51.100.0/24",
"203.0.113.0/24", "255.255.255.255/32", "100.64.0.0/10", "192.88.99.0/24", "240.0.0.0/4");
Expand Down Expand Up @@ -445,4 +448,12 @@ public boolean getHandleApisCaseInsensitive() {
public void setHandleApisCaseInsensitive(boolean handleApisCaseInsensitive) {
this.handleApisCaseInsensitive = handleApisCaseInsensitive;
}

public boolean getAllowOptionsAPIs() {
return allowOptionsAPIs;
}

public void setAllowOptionsAPIs(boolean allowOptionsAPIs) {
this.allowOptionsAPIs = allowOptionsAPIs;
}
}
14 changes: 13 additions & 1 deletion libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@ public class BackwardCompatibility {
public static final String MOVE_AZURE_SAML = "moveAzureSamlToNormalSaml";
private int moveAzureSamlToNormalSaml;

public static final String DELETE_OPTIONS_API = "deleteOptionsAPIs";
private int deleteOptionsAPIs;

public BackwardCompatibility(int id, int dropFilterSampleData, int resetSingleTypeInfoCount, int dropWorkflowTestResult,
int readyForNewTestingFramework,int addAktoDataTypes, boolean deploymentStatusUpdated,
int authMechanismData, boolean mirroringLambdaTriggered, int deleteAccessListFromApiToken,
Expand All @@ -106,7 +109,7 @@ public BackwardCompatibility(int id, int dropFilterSampleData, int resetSingleTy
int loginSignupGroups, int vulnerableApiUpdationVersionV1, int riskScoreGroups,
int deactivateCollections, int disableAwsSecretPii, int apiCollectionAutomatedField,
int automatedApiGroups, int addAdminRoleIfAbsent, int dropSpecialCharacterApiCollections, int fixApiAccessType,
int addDefaultFilters, int moveAzureSamlToNormalSaml) {
int addDefaultFilters, int moveAzureSamlToNormalSaml, int deleteOptionsAPIs) {
this.id = id;
this.dropFilterSampleData = dropFilterSampleData;
this.resetSingleTypeInfoCount = resetSingleTypeInfoCount;
Expand Down Expand Up @@ -137,6 +140,7 @@ public BackwardCompatibility(int id, int dropFilterSampleData, int resetSingleTy
this.dropSpecialCharacterApiCollections = dropSpecialCharacterApiCollections;
this.fixApiAccessType = fixApiAccessType;
this.moveAzureSamlToNormalSaml = moveAzureSamlToNormalSaml;
this.deleteOptionsAPIs = deleteOptionsAPIs;
}

public BackwardCompatibility() {
Expand Down Expand Up @@ -413,4 +417,12 @@ public int getMoveAzureSamlToNormalSaml() {
public void setMoveAzureSamlToNormalSaml(int moveAzureSamlToNormalSaml) {
this.moveAzureSamlToNormalSaml = moveAzureSamlToNormalSaml;
}

public int getDeleteOptionsAPIs() {
return deleteOptionsAPIs;
}

public void setDeleteOptionsAPIs(int deleteOptionsAPIs) {
this.deleteOptionsAPIs = deleteOptionsAPIs;
}
}

0 comments on commit 6c1ac0d

Please sign in to comment.