diff --git a/apps/dashboard/src/main/java/com/akto/action/InternalAction.java b/apps/dashboard/src/main/java/com/akto/action/InternalAction.java index fc59ae3cc9..b1f3ef6c76 100644 --- a/apps/dashboard/src/main/java/com/akto/action/InternalAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/InternalAction.java @@ -1,34 +1,33 @@ package com.akto.action; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import org.bson.Document; import org.bson.conversions.Bson; +import com.akto.dao.ApiCollectionsDao; import com.akto.dao.SingleTypeInfoDao; import com.akto.dao.context.Context; +import com.akto.dto.ApiCollection; import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.User; import com.akto.dto.traffic.Key; -import com.akto.dto.type.SingleTypeInfo; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; import com.akto.usage.UsageMetricCalculator; -import com.mongodb.client.model.Filters; public class InternalAction extends UserAction { private static final LoggerMaker loggerMaker = new LoggerMaker(InternalAction.class, LogDb.DASHBOARD); - String headerKey; boolean actuallyDelete; int count; + int timestamp; - public String deleteApisBasedOnHeader() { - - if (headerKey == null || headerKey.isEmpty()) { - addActionError("Invalid header key"); - return ERROR.toUpperCase(); - } + public String deleteApisWithoutHost() { User user = getSUser(); if (user.getLogin() != null && !user.getLogin().contains("@akto")) { @@ -36,48 +35,77 @@ public String deleteApisBasedOnHeader() { return ERROR.toUpperCase(); } - int time = Context.now(); - Bson filter = Filters.and(Filters.eq(SingleTypeInfo._PARAM, headerKey), - UsageMetricCalculator.excludeDemosAndDeactivated(SingleTypeInfo._API_COLLECTION_ID)); - loggerMaker.infoAndAddToDb("Executing deleteApisBasedOnHeader find query"); - List apiList = SingleTypeInfoDao.instance.fetchEndpointsInCollection(filter); - - int delta = Context.now() - time; - loggerMaker.infoAndAddToDb("Finished deleteApisBasedOnHeader find query " + delta); + List apiCollections = ApiCollectionsDao.fetchAllHosts(); - if (apiList != null && !apiList.isEmpty()) { + Set keys = new HashSet<>(); + int time = Context.nowInMillis(); + int delta = Context.nowInMillis(); + Set demosAndDeactivated = UsageMetricCalculator.getDemosAndDeactivated(); + for (ApiCollection apiCollection : apiCollections) { - List keys = new ArrayList<>(); - for (ApiInfoKey apiInfoKey : apiList) { - loggerMaker.infoAndAddToDb("deleteApisBasedOnHeader " + apiInfoKey.toString()); - keys.add(new Key(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod(), -1, 0, - 0)); + + int apiCollectionId = apiCollection.getId(); + if (demosAndDeactivated.contains(apiCollectionId)) { + loggerMaker.infoAndAddToDb("Skipping deleteApisBasedOnHeader for apiCollectionId " + apiCollectionId); + continue; } - - count = apiList.size(); - if (actuallyDelete) { - try { - time = Context.now(); - loggerMaker.infoAndAddToDb("deleteApisBasedOnHeader deleting APIs"); - com.akto.utils.Utils.deleteApis(keys); - delta = Context.now() - time; - loggerMaker.infoAndAddToDb("deleteApisBasedOnHeader deleted APIs " + delta); - } catch (Exception e) { - e.printStackTrace(); - addActionError("Error deleting APIs"); - return ERROR.toUpperCase(); + + List pipeline = Arrays.asList(new Document("$match", + new Document("$and", Arrays.asList(new Document("$or", Arrays.asList( + new Document("lastSeen", new Document("$gt", timestamp)), + new Document("timestamp", new Document("$gt", timestamp)))), + new Document("apiCollectionId", apiCollectionId)))), + new Document("$group", + new Document("_id", + new Document("apiCollectionId", "$apiCollectionId") + .append("url", "$url") + .append("method", "$method")) + .append("hasHostParam", + new Document("$max", + new Document("$cond", + Arrays.asList(new Document("$eq", + Arrays.asList("$param", "host")), 1L, 0L)))) + .append("documents", + new Document("$push", "$$ROOT"))), + new Document("$match", + new Document("hasHostParam", + new Document("$eq", 0L))), + new Document("$project", + new Document("_id", 1L))); + + time = Context.nowInMillis(); + loggerMaker.infoAndAddToDb("Executing deleteApisBasedOnHeader find query " + apiCollectionId); + List apiList = SingleTypeInfoDao.instance.processPipelineForEndpoint(pipeline); + delta = Context.nowInMillis() - time; + loggerMaker.infoAndAddToDb("Finished deleteApisBasedOnHeader find query " + delta + " " + apiCollectionId); + if (apiList != null && !apiList.isEmpty()) { + for (ApiInfoKey apiInfoKey : apiList) { + loggerMaker.infoAndAddToDb("deleteApisBasedOnHeader " + apiInfoKey.toString()); + keys.add( + new Key(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod(), -1, 0, + 0)); } } } - return SUCCESS.toUpperCase(); - } - public String getHeaderKey() { - return headerKey; - } + count = keys.size(); + if (keys != null && !keys.isEmpty() && actuallyDelete) { + try { + time = Context.nowInMillis(); + loggerMaker.infoAndAddToDb("deleteApisBasedOnHeader deleting APIs"); + List keyList = new ArrayList<>(); + keyList.addAll(keys); + com.akto.utils.Utils.deleteApis(keyList); + delta = Context.nowInMillis() - time; + loggerMaker.infoAndAddToDb("deleteApisBasedOnHeader deleted APIs " + delta); + } catch (Exception e) { + e.printStackTrace(); + addActionError("Error deleting APIs"); + return ERROR.toUpperCase(); + } + } - public void setHeaderKey(String headerKey) { - this.headerKey = headerKey; + return SUCCESS.toUpperCase(); } public boolean getActuallyDelete() { @@ -95,4 +123,12 @@ public int getCount() { public void setCount(int count) { this.count = count; } + + public int getTimestamp() { + return timestamp; + } + + public void setTimestamp(int timestamp) { + this.timestamp = timestamp; + } } diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index 611bd05a8f..4cf38704bb 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -2060,7 +2060,7 @@ - + diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java index 8a32ebc9e0..54dd13c3a8 100644 --- a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java @@ -365,12 +365,6 @@ public List fetchEndpointsInCollection(int apiCollectionId) return processPipelineForEndpoint(pipeline); } - public List fetchEndpointsInCollection(Bson filter) { - List pipeline = getPipelineForEndpoints(filter); - pipeline.add(Aggregates.limit(SingleTypeInfoDao.LARGE_LIMIT)); - return processPipelineForEndpoint(pipeline); - } - public List fetchEndpointsInCollection(Method method) { Bson filter = null; if (method == null) { @@ -412,7 +406,7 @@ private List getPipelineForEndpoints(Bson matchCriteria) { return pipeline; } - private List processPipelineForEndpoint(List pipeline){ + public List processPipelineForEndpoint(List pipeline){ MongoCursor endpointsCursor = instance.getMCollection().aggregate(pipeline, BasicDBObject.class).cursor(); List endpoints = new ArrayList<>();