Skip to content

Commit

Permalink
Merge pull request #1572 from akto-api-security/feature/reset_data_type
Browse files Browse the repository at this point in the history 
add reset job for data types
  • Loading branch information
@notshivansh
notshivansh authored Oct 1, 2024
2 parents a1d467e + ad39dfa commit a43f12c
Show file tree
Hide file tree
Showing 4 changed files with 145 additions and 1 deletion.
110 changes: 109 additions & 1 deletion apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@
import com.akto.dto.traffic.Key;
import com.akto.dto.traffic.SampleData;
import com.akto.dto.type.SingleTypeInfo;
import com.akto.listener.RuntimeListener;
import com.akto.log.LoggerMaker;
import com.akto.log.LoggerMaker.LogDb;
import com.akto.parsers.HttpCallParser;
import com.akto.util.JSONUtils;
import com.akto.utils.AccountHTTPCallParserAktoPolicyInfo;
import com.akto.utils.AktoCustomException;
import com.akto.utils.RedactSampleData;
import com.fasterxml.jackson.core.JsonFactory;
Expand Down Expand Up @@ -40,7 +42,7 @@
import static com.akto.utils.Utils.extractJsonResponse;

public class CustomDataTypeAction extends UserAction{
private static LoggerMaker loggerMaker = new LoggerMaker(CustomDataTypeAction.class);
private static final LoggerMaker loggerMaker = new LoggerMaker(CustomDataTypeAction.class, LogDb.DASHBOARD);

private static final ExecutorService service = Executors.newFixedThreadPool(1);
private boolean createNew;
Expand Down Expand Up @@ -507,6 +509,112 @@ public void setValue(String value) {
private long totalSampleDataCount;
private long currentProcessed;
private static final int pageSize = 1000;

public String resetDataTypeRetro() {
customDataType = CustomDataTypeDao.instance.findOne(CustomDataType.NAME, name);
aktoDataType = AktoDataTypeDao.instance.findOne(AktoDataType.NAME, name);

if (customDataType == null && aktoDataType == null) {
addActionError("Data type does not exist");
return ERROR.toUpperCase();
}

/*
* we find all samples which contain the data type
* and re-run runtime on the samples.
*/

int accountId = Context.accountId.get();
service.submit(() -> {
Context.accountId.set(accountId);
loggerMaker.infoAndAddToDb("Triggered a job to recalculate data types");
int skip = 0;
final int LIMIT = 100;
final int BATCH_SIZE = 200;
List<SampleData> sampleDataList = new ArrayList<>();
Bson sort = Sorts.ascending("_id.apiCollectionId", "_id.url", "_id.method");
List<HttpResponseParams> responses = new ArrayList<>();
this.customSubTypeMatches = new ArrayList<>();
do {
sampleDataList = SampleDataDao.instance.findAll(Filters.empty(), skip, LIMIT, sort);
skip += LIMIT;
for(SampleData sampleData : sampleDataList){
List<String> samples = sampleData.getSamples();
Set<String> foundSet = new HashSet<>();
for (String sample: samples) {
Key apiKey = sampleData.getId();
try {
HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample);
boolean skip1=false, skip2=false, skip3=false, skip4=false;
try {
skip1 = forHeaders(httpResponseParams.getHeaders(), customDataType, apiKey);
skip2 = forHeaders(httpResponseParams.requestParams.getHeaders(), customDataType, apiKey);
} catch (Exception e) {
}
try {
skip3 = forPayload(httpResponseParams.getPayload(), customDataType, apiKey);
skip4 = forPayload(httpResponseParams.requestParams.getPayload(), customDataType, apiKey);
} catch (Exception e) {
}
String key = skip1 + " " + skip2 + " " + skip3 + " " + skip4 + " " + sampleData.getId().toString();
if ((skip1 || skip2 || skip3 || skip4) && !foundSet.contains(key)) {
foundSet.add(key);
responses.add(httpResponseParams);
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
if (sampleDataList != null) {
loggerMaker.infoAndAddToDb("Read sampleData to recalculate data types " + sampleDataList.size());
}

if (!responses.isEmpty() && responses.size() >= BATCH_SIZE) {
loggerMaker.infoAndAddToDb(
"Starting processing responses to recalculate data types " + responses.size());
SingleTypeInfo.fetchCustomDataTypes(accountId);
AccountHTTPCallParserAktoPolicyInfo info = RuntimeListener.accountHTTPParserMap.get(accountId);
if (info == null) {
info = new AccountHTTPCallParserAktoPolicyInfo();
HttpCallParser callParser = new HttpCallParser("userIdentifier", 1, 1, 1, false);
info.setHttpCallParser(callParser);
RuntimeListener.accountHTTPParserMap.put(accountId, info);
}
AccountSettings accountSettings = AccountSettingsDao.instance
.findOne(AccountSettingsDao.generateFilter());
responses = com.akto.runtime.Main.filterBasedOnHeaders(responses, accountSettings);
info.getHttpCallParser().syncFunction(responses, true, false, accountSettings);
info.getHttpCallParser().apiCatalogSync.buildFromDB(false, false);
loggerMaker.infoAndAddToDb("Processed responses to recalculate data types " + responses.size());
responses.clear();
}

}while(sampleDataList!=null && !sampleDataList.isEmpty());

if (!responses.isEmpty()) {
loggerMaker.infoAndAddToDb("Starting processing responses to recalculate data types " + responses.size());
SingleTypeInfo.fetchCustomDataTypes(accountId);
AccountHTTPCallParserAktoPolicyInfo info = RuntimeListener.accountHTTPParserMap.get(accountId);
if (info == null) {
info = new AccountHTTPCallParserAktoPolicyInfo();
HttpCallParser callParser = new HttpCallParser("userIdentifier", 1, 1, 1, false);
info.setHttpCallParser(callParser);
RuntimeListener.accountHTTPParserMap.put(accountId, info);
}
AccountSettings accountSettings = AccountSettingsDao.instance
.findOne(AccountSettingsDao.generateFilter());
responses = com.akto.runtime.Main.filterBasedOnHeaders(responses, accountSettings);
info.getHttpCallParser().syncFunction(responses, true, false, accountSettings);
info.getHttpCallParser().apiCatalogSync.buildFromDB(false, false);
loggerMaker.infoAndAddToDb("Processed responses to recalculate data types " + responses.size());
}
loggerMaker.infoAndAddToDb("Finished a job to recalculate data types");
});

return SUCCESS.toUpperCase();
}

public String reviewCustomDataType() {
customSubTypeMatches = new ArrayList<>();
CustomDataType customDataType;
Expand Down
20 changes: 20 additions & 0 deletions apps/dashboard/src/main/resources/struts.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2566,6 +2566,26 @@
</result>
</action>

<action name="api/resetDataTypeRetro" class="com.akto.action.CustomDataTypeAction" method="resetDataTypeRetro">
<interceptor-ref name="json"/>
<interceptor-ref name="defaultStack" />
<interceptor-ref name="roleAccessInterceptor">
<param name="featureLabel">SENSITIVE_DATA</param>
<param name="accessType">READ_WRITE</param>
</interceptor-ref>
<result name="FORBIDDEN" type="json">
<param name="statusCode">403</param>
<param name="ignoreHierarchy">false</param>
<param name="includeProperties">^actionErrors.*</param>
</result>
<result name="SUCCESS" type="json"/>
<result name="ERROR" type="json">
<param name="statusCode">422</param>
<param name="ignoreHierarchy">false</param>
<param name="includeProperties">^actionErrors.*</param>
</result>
</action>

<action name="api/toggleDataTypeActiveParam" class="com.akto.action.CustomDataTypeAction" method="toggleDataTypeActiveParam">
<interceptor-ref name="json"/>
<interceptor-ref name="defaultStack" />
Expand Down
6 changes: 6 additions & 0 deletions ..._web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,14 @@ const handleRedirect = () => {
navigate("/dashboard/observe/data-types")
}

const handleReset = async () => {
await api.resetDataTypeRetro(subType)
func.setToast(true, false, "Resetting data types")
}

const primaryActions = (
<HorizontalStack gap={"2"}>
<Button id={"reset-data-type"} onClick={handleReset}>Reset</Button>
<DateRangeFilter initialDispatch = {currDateRange} dispatch={(dateObj) => dispatchCurrDateRange({type: "update", period: dateObj.period, title: dateObj.title, alias: dateObj.alias})}/>
<Button id={"all-data-types"} primary onClick={handleRedirect}>Create custom data types</Button>
</HorizontalStack>
Expand Down
10 changes: 10 additions & 0 deletions apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -733,5 +733,15 @@ export default {
apiCollectionId: apiInfoKey.apiCollectionId
}
})
},
resetDataTypeRetro(name){
return request({
url: '/api/resetDataTypeRetro',
method: 'post',
data: {
name: name,
}
})
}

}

0 comments on commit a43f12c

Please sign in to comment.