Skip to content

Commit

Permalink
Merge pull request #1967 from akto-api-security/feature/save_vuln_in_…
Browse files Browse the repository at this point in the history 
…new_collection

Feature/save vuln in new collection
  • Loading branch information
@Ark2307
Ark2307 authored Jan 14, 2025
2 parents d080dbb + b23ebd6 commit d7556cb
Show file tree
Hide file tree
Showing 16 changed files with 389 additions and 52 deletions.
46 changes: 34 additions & 12 deletions apps/dashboard/src/main/java/com/akto/action/testing/StartTestAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -614,7 +614,7 @@ private Map<String, Integer> getCountMapForQueryMode(ObjectId testingRunResultSu
Map<String, Integer> resultantMap = new HashMap<>();

List<Bson> filterList = prepareTestRunResultsFilters(testingRunResultSummaryId, queryMode);
int count = (int) TestingRunResultDao.instance.count(Filters.and(filterList));
int count = VulnerableTestingRunResultDao.instance.countFromDb(Filters.and(filterList), queryMode.equals(QueryMode.VULNERABLE));
resultantMap.put(queryMode.toString(), count);

return resultantMap;
Expand Down Expand Up @@ -720,8 +720,8 @@ public String fetchTestingRunResults() {

timeNow = Context.now();
Bson filters = testingRunResultFilters.isEmpty() ? Filters.empty() : Filters.and(testingRunResultFilters);
this.testingRunResults = TestingRunResultDao.instance
.fetchLatestTestingRunResultWithCustomAggregations(filters, pageLimit, skip, sortStage);
this.testingRunResults = VulnerableTestingRunResultDao.instance
.fetchLatestTestingRunResultWithCustomAggregations(filters, pageLimit, skip, sortStage, testingRunResultSummaryId, queryMode.equals(QueryMode.VULNERABLE));
loggerMaker.infoAndAddToDb("[" + (Context.now() - timeNow) + "] Fetched testing run results of size: " + testingRunResults.size(), LogDb.DASHBOARD);

timeNow = Context.now();
Expand Down Expand Up @@ -778,13 +778,25 @@ public String fetchVulnerableTestRunResults() {
try {
testingRunResultSummaryId = new ObjectId(testingRunResultSummaryHexId);
Bson filterForReport = com.akto.action.testing.Utils.createFiltersForTestingReport(reportFilterList);
boolean isStoredInVulnerableCollection = VulnerableTestingRunResultDao.instance.isStoredInVulnerableCollection(testingRunResultSummaryId, true);
Bson filters = Filters.and(
Filters.eq(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, testingRunResultSummaryId),
Filters.eq(TestingRunResult.VULNERABLE, true),
filterForReport
);
List<TestingRunResult> testingRunResultList = new ArrayList<>();
if(isStoredInVulnerableCollection){
filters = Filters.and(
Filters.eq(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, testingRunResultSummaryId),
filterForReport
);
testingRunResultList = VulnerableTestingRunResultDao.instance.findAll(filters, skip, 50, null);
}else{
testingRunResultList = TestingRunResultDao.instance.findAll(filters, skip, 50, null);
}


List<TestingRunResult> testingRunResultList = TestingRunResultDao.instance.findAll(filters, skip, 50, null);

// Map<String, String> sampleDataVsCurlMap = new HashMap<>();
// for (TestingRunResult runResult: testingRunResultList) {
// WorkflowTest workflowTest = runResult.getWorkflowTest();
Expand Down Expand Up @@ -848,7 +860,7 @@ public static String getNodeResultLastMessage(String message) {

public String fetchTestRunResultDetails() {
ObjectId testingRunResultId = new ObjectId(testingRunResultHexId);
this.testingRunResult = TestingRunResultDao.instance.findOne("_id", testingRunResultId);
this.testingRunResult = VulnerableTestingRunResultDao.instance.findOneWithComparison(Filters.eq(Constants.ID, testingRunResultId), null);
List<GenericTestResult> runResults = new ArrayList<>();

for (GenericTestResult testResult: this.testingRunResult.getTestResults()) {
Expand All @@ -868,7 +880,7 @@ public String fetchTestRunResultDetails() {

public String fetchIssueFromTestRunResultDetails() {
ObjectId testingRunResultId = new ObjectId(testingRunResultHexId);
TestingRunResult result = TestingRunResultDao.instance.findOne(Constants.ID, testingRunResultId);
TestingRunResult result = VulnerableTestingRunResultDao.instance.findOneWithComparison(Filters.eq(Constants.ID, testingRunResultId), null);
try {
if (result.isVulnerable()) {
// name = category
Expand Down Expand Up @@ -1132,12 +1144,14 @@ public void run() {
Context.accountId.set(accountId);
try {
ObjectId summaryObjectId = new ObjectId(testingRunResultSummaryHexId);
List<TestingRunResult> testingRunResults = TestingRunResultDao.instance.findAll(
boolean isStoredInVulnerableCollection = VulnerableTestingRunResultDao.instance.isStoredInVulnerableCollection(summaryObjectId, true);
List<TestingRunResult> testingRunResults = VulnerableTestingRunResultDao.instance.findAll(
Filters.and(
Filters.eq(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, summaryObjectId),
vulnerableFilter
),
Projections.include(TestingRunResult.API_INFO_KEY, TestingRunResult.TEST_SUB_TYPE)
Projections.include(TestingRunResult.API_INFO_KEY, TestingRunResult.TEST_SUB_TYPE),
isStoredInVulnerableCollection
);

if(testingRunResults.isEmpty()){
Expand Down Expand Up @@ -1182,10 +1196,18 @@ public void run() {
);

// update testing run results, by setting them isIgnored true
TestingRunResultDao.instance.updateMany(
Filters.in(Constants.ID, ignoredResults),
Updates.set(TestingRunResult.IS_IGNORED_RESULT, true)
);
if(isStoredInVulnerableCollection){
VulnerableTestingRunResultDao.instance.updateMany(
Filters.in(Constants.ID, ignoredResults),
Updates.set(TestingRunResult.IS_IGNORED_RESULT, true)
);
}else{
TestingRunResultDao.instance.updateMany(
Filters.in(Constants.ID, ignoredResults),
Updates.set(TestingRunResult.IS_IGNORED_RESULT, true)
);
}

} catch (Exception e) {
e.printStackTrace();
}
Expand Down
95 changes: 78 additions & 17 deletions apps/dashboard/src/main/java/com/akto/action/testing_issues/IssuesAction.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.akto.action.testing_issues;

import com.akto.action.ExportSampleDataAction;
import com.akto.action.UserAction;
import com.akto.action.testing.Utils;
import com.akto.dao.HistoricalDataDao;
Expand All @@ -11,6 +10,7 @@
import com.akto.dao.test_editor.YamlTemplateDao;
import com.akto.dao.testing.TestingRunResultDao;
import com.akto.dao.testing.TestingRunResultSummariesDao;
import com.akto.dao.testing.VulnerableTestingRunResultDao;
import com.akto.dao.testing.sources.TestReportsDao;
import com.akto.dao.testing.sources.TestSourceConfigsDao;
import com.akto.dao.testing_run_findings.TestingRunIssuesDao;
Expand Down Expand Up @@ -328,13 +328,33 @@ public String fetchVulnerableTestingRunResultsFromIssues() {
issues = TestingRunIssuesDao.instance.findAll(filters, skip, 50, null);
}
List<Bson> andFilters = new ArrayList<>();
List<Bson> filtersForNewCollection = new ArrayList<>();

Map<String,Boolean> summaryIdVsIsNew = new HashMap<>();

for (TestingRunIssues issue : issues) {
andFilters.add(Filters.and(
Filters.eq(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, issue.getLatestTestingRunSummaryId()),
Filters.eq(TestingRunResult.TEST_SUB_TYPE, issue.getId().getTestSubCategory()),
Filters.eq(TestingRunResult.API_INFO_KEY, issue.getId().getApiInfoKey()),
Filters.eq(TestingRunResult.VULNERABLE, true)
));

ObjectId currentSummaryId = issue.getLatestTestingRunSummaryId();

Bson baseFilter = Filters.and(
Filters.eq(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, currentSummaryId),
Filters.eq(TestingRunResult.TEST_SUB_TYPE, issue.getId().getTestSubCategory()),
Filters.eq(TestingRunResult.API_INFO_KEY, issue.getId().getApiInfoKey())
);

Boolean val = summaryIdVsIsNew.get(currentSummaryId.toHexString());
if(val == null){
val = VulnerableTestingRunResultDao.instance.isStoredInVulnerableCollection(currentSummaryId, true);
summaryIdVsIsNew.put(currentSummaryId.toHexString(), val);
}

if(!val){
andFilters.add(
Filters.and(baseFilter,Filters.eq(TestingRunResult.VULNERABLE, true))
);
}else{
filtersForNewCollection.add(baseFilter);
}
}
if (issues.isEmpty()) {
this.testingRunResults = new ArrayList<>();
Expand All @@ -348,9 +368,17 @@ public String fetchVulnerableTestingRunResultsFromIssues() {
String key = issue.getId().getApiInfoKey().toString() + "_" + testSubCategory;
testingRunIssuesMap.put(key, issue);
}
if(!andFilters.isEmpty()){
Bson orFilters = Filters.or(andFilters);
this.testingRunResults = TestingRunResultDao.instance.findAll(orFilters);
}

if (!filtersForNewCollection.isEmpty()) {
this.testingRunResults.addAll(
VulnerableTestingRunResultDao.instance.findAll(Filters.or(filtersForNewCollection))
);
}

Bson orFilters = Filters.or(andFilters);
this.testingRunResults = TestingRunResultDao.instance.findAll(orFilters);
Map<String, String> sampleDataVsCurlMap = new HashMap<>();
// todo: fix
for (TestingRunResult runResult: this.testingRunResults) {
Expand Down Expand Up @@ -426,7 +454,7 @@ public String fetchTestingRunResult() {
Filters.eq(TestingRunResult.TEST_SUB_TYPE, testSubType),
Filters.eq(TestingRunResult.API_INFO_KEY, issue.getId().getApiInfoKey())
);
testingRunResult = TestingRunResultDao.instance.findOne(filterForRunResult);
testingRunResult = VulnerableTestingRunResultDao.instance.findOneWithComparison(filterForRunResult, null);
if (issue.isUnread() && (currentUserRole.equals(Role.ADMIN) || currentUserRole.equals(Role.MEMBER))) {
logger.info("Issue id from db to be marked as read " + issueId);
Bson update = Updates.combine(Updates.set(TestingRunIssues.UNREAD, false),
Expand Down Expand Up @@ -574,8 +602,14 @@ public void run() {
countIssuesMap.put(Severity.LOW.toString(), 0);

// update summaries accordingly with issues ignored

Map<ObjectId,String> mapSummaryToResultId = TestingRunResultDao.instance.mapSummaryIdToTestingResultHexId(testingRunResultHexIdsMap.keySet());
// currently we change the summaries from result page only
// so only 1 result comes at a time
// Map<String,String> testingRunResultHexIdsMap has only 1 result.

Map<ObjectId,String> mapSummaryToResultId = VulnerableTestingRunResultDao.instance.mapSummaryIdToTestingResultHexId(testingRunResultHexIdsMap.keySet());
if(mapSummaryToResultId.isEmpty()){
mapSummaryToResultId = TestingRunResultDao.instance.mapSummaryIdToTestingResultHexId(testingRunResultHexIdsMap.keySet());
}
Map<ObjectId,Map<String,Integer>> summaryWiseCountMap = new HashMap<>();

for(ObjectId summaryId: mapSummaryToResultId.keySet()){
Expand Down Expand Up @@ -604,21 +638,44 @@ public void run() {
List<String> issueStatusQuery;
List<TestingRunResult> testingRunResultList;
private Map<String, List<String>> filters;

public String fetchIssuesByStatusAndSummaryId() {
if(latestTestingRunSummaryId == null || latestTestingRunSummaryId.isEmpty()){
addActionError("SummaryId is a required field and cannot be empty.");
return ERROR.toUpperCase();
}
if(!ObjectId.isValid(latestTestingRunSummaryId)){
addActionError("SummaryId is not valid");
return ERROR.toUpperCase();
}

ObjectId objectId = new ObjectId(latestTestingRunSummaryId);

Bson triFilters = Filters.and(
Filters.in(TestingRunIssues.TEST_RUN_ISSUES_STATUS, issueStatusQuery),
Filters.in(TestingRunIssues.LATEST_TESTING_RUN_SUMMARY_ID, new ObjectId(latestTestingRunSummaryId))
Filters.in(TestingRunIssues.LATEST_TESTING_RUN_SUMMARY_ID, objectId)
);
issues = TestingRunIssuesDao.instance.findAll(triFilters, Projections.include("_id"));

List<Bson> testingRunResultsFilterList = new ArrayList<>();
boolean isStoredInVulnerableCollection = VulnerableTestingRunResultDao.instance.isStoredInVulnerableCollection(objectId, true);
for(TestingRunIssues issue: issues) {
testingRunResultsFilterList.add(Filters.and(
Bson filter = Filters.empty();
if(isStoredInVulnerableCollection){
filter = Filters.and(
Filters.eq(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, new ObjectId(latestTestingRunSummaryId)),
Filters.eq(TestingRunResult.API_INFO_KEY, issue.getId().getApiInfoKey()),
Filters.eq(TestingRunResult.TEST_SUB_TYPE, issue.getId().getTestSubCategory())
);

}else{
filter = Filters.and(
Filters.eq(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, new ObjectId(latestTestingRunSummaryId)),
Filters.eq(TestingRunResult.VULNERABLE, true),
Filters.eq(TestingRunResult.API_INFO_KEY, issue.getId().getApiInfoKey()),
Filters.eq(TestingRunResult.TEST_SUB_TYPE, issue.getId().getTestSubCategory())
));
);
}
testingRunResultsFilterList.add(filter);
}

List<Bson> filtersList = new ArrayList<>();
Expand All @@ -632,7 +689,11 @@ public String fetchIssuesByStatusAndSummaryId() {
return SUCCESS.toUpperCase();
}

testingRunResultList = TestingRunResultDao.instance.fetchLatestTestingRunResultWithCustomAggregations(Filters.and(filtersList), limit, skip, sortStage);
if(isStoredInVulnerableCollection){
testingRunResultList = VulnerableTestingRunResultDao.instance.fetchLatestTestingRunResultWithCustomAggregations(Filters.and(filtersList), limit, skip, sortStage);
}else{
testingRunResultList = TestingRunResultDao.instance.fetchLatestTestingRunResultWithCustomAggregations(Filters.and(filtersList), limit, skip, sortStage);
}

return SUCCESS.toUpperCase();
}
Expand Down
21 changes: 21 additions & 0 deletions apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2975,11 +2975,32 @@ private static void moveOktaOidcSSO(BackwardCompatibility backwardCompatibility)
}
}

private static void markSummariesAsVulnerable(BackwardCompatibility backwardCompatibility){
// case for the customers where vulnerable are stored in new collection and only testing runs are marked as new.

if(backwardCompatibility.getMarkSummariesVulnerable() == 0){

List<ObjectId> summaryIds = VulnerableTestingRunResultDao.instance.summaryIdsStoredForVulnerableTests();
if(!summaryIds.isEmpty()){
TestingRunResultSummariesDao.instance.updateMany(
Filters.in(Constants.ID, summaryIds),
Updates.set(TestingRunResultSummary.IS_NEW_TESTING_RUN_RESULT_SUMMARY, true)
);
}

BackwardCompatibilityDao.instance.updateOne(
Filters.eq("_id", backwardCompatibility.getId()),
Updates.set(BackwardCompatibility.MARK_SUMMARIES_NEW_FOR_VULNERABLE, Context.now())
);
}
}

public static void setBackwardCompatibilities(BackwardCompatibility backwardCompatibility){
if (DashboardMode.isMetered()) {
initializeOrganizationAccountBelongsTo(backwardCompatibility);
setOrganizationsInBilling(backwardCompatibility);
}
markSummariesAsVulnerable(backwardCompatibility);
setAktoDefaultNewUI(backwardCompatibility);
dropLastCronRunInfoField(backwardCompatibility);
fetchIntegratedConnections(backwardCompatibility);
Expand Down
Loading

0 comments on commit d7556cb

Please sign in to comment.