diff --git a/src/main/java/com/uid2/operator/model/IdentityMapV2Input.java b/src/main/java/com/uid2/operator/model/IdentityMapV2Input.java new file mode 100644 index 000000000..9b8437ba0 --- /dev/null +++ b/src/main/java/com/uid2/operator/model/IdentityMapV2Input.java @@ -0,0 +1,12 @@ +package com.uid2.operator.model; + +import com.uid2.operator.service.InputUtil; + +import java.util.Objects; + +public record IdentityMapV2Input(String diiType, InputUtil.InputVal[] inputList) { + public IdentityMapV2Input { + Objects.requireNonNull(diiType); + Objects.requireNonNull(inputList); + } +} diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 503ac9ace..2481b1019 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -107,16 +107,32 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final UidInstanceIdProvider uidInstanceIdProvider; protected IUIDOperatorService idService; + // uid2_operator_identity_map_inputs private final Map _identityMapMetricSummaries = new HashMap<>(); + // uid2_operator_identity_map_services_inputs + private final Map _identityMapServicesMetricSummaries = new HashMap<>(); + // uid2_token_refresh_duration_seconds private final Map, DistributionSummary> _refreshDurationMetricSummaries = new HashMap<>(); + // uid2_advertising_token_expired_on_refresh_total private final Map, Counter> _advertisingTokenExpiryStatus = new HashMap<>(); + // uid2_token_generate_tcf_usage_total private final Map _tokenGenerateTCFUsage = new HashMap<>(); + // uid2_operator_identity_map_unmapped_total (reason=invalid, reason=optout) private final Map> _identityMapUnmappedIdentifiers = new HashMap<>(); + // uid2_operator_identity_map_services_unmapped_total (reason=invalid, reason=optout) + private final Map> _identityMapServicesUnmappedIdentifiers = new HashMap<>(); + // uid2_operator_identity_map_unmapped_requests_total private final Map _identityMapRequestWithUnmapped = new HashMap<>(); + // uid2_client_sdk_versions_total private final Map, Counter> _clientVersions = new HashMap<>(); + // uid2_token_validate_total private final Map, Counter> _tokenValidateCounters = new HashMap<>(); - private final Map optOutStatusCounters = new HashMap<>(); + // uid2_operator_optout_status_input_size + private final Map optOutStatusInputSizeCounters = new HashMap<>(); + // uid2_operator_optout_status_optout_size + private final Map optOutStatusOptOutSizeCounters = new HashMap<>(); + private final IdentityScope identityScope; private final V2PayloadHandler encryptedPayloadHandler; private final boolean phoneSupport; @@ -1097,17 +1113,18 @@ private boolean isTokenInputValid(InputUtil.InputVal input, RoutingContext rc) { return true; } - private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal[] inputList) { + private JsonObject processIdentityMapV2Response(RoutingContext rc, IdentityMapV2Input v2Input) { RuntimeConfig config = getConfigFromRc(rc); IdentityEnvironment env = config.getIdentityEnvironment(); + InputUtil.InputVal[] inputList = v2Input.inputList(); final Instant now = Instant.now(); final JsonArray mapped = new JsonArray(); final JsonArray unmapped = new JsonArray(); - final int count = inputList.length; + final int inputCount = inputList.length; int invalidCount = 0; int optoutCount = 0; - for (int i = 0; i < count; ++i) { + for (int i = 0; i < inputCount; ++i) { final InputUtil.InputVal input = inputList[i]; if (input != null && input.isValid()) { final MappedIdentity mappedIdentity = idService.mapIdentity( @@ -1138,7 +1155,7 @@ private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal } } - recordIdentityMapStats(rc, inputList.length, invalidCount, optoutCount); + recordIdentityMapStats(rc, Map.of(v2Input.diiType(), inputCount), invalidCount, optoutCount); final JsonObject resp = new JsonObject(); resp.put("mapped", mapped); @@ -1154,12 +1171,12 @@ private JsonObject processIdentityMapV3Response(RoutingContext rc, Map diiTypeCounts = new HashMap<>(); for (Map.Entry identityType : input.entrySet()) { JsonArray mappedIdentityList = new JsonArray(); final InputUtil.InputVal[] rawIdentityList = identityType.getValue(); - inputTotalCount += rawIdentityList.length; + diiTypeCounts.put(identityType.getKey(), rawIdentityList.length); for (final InputUtil.InputVal rawId : rawIdentityList) { final JsonObject resp = new JsonObject(); @@ -1187,7 +1204,7 @@ private JsonObject processIdentityMapV3Response(RoutingContext rc, Map getInputList = null; + Supplier getV2Request = null; final JsonArray emails = JsonParseUtils.parseArray(obj, "email", rc); if (emails != null && !emails.isEmpty()) { - getInputList = () -> createInputList(emails, IdentityType.Email, InputUtil.IdentityInputType.Raw); + getV2Request = () -> new IdentityMapV2Input("email", createInputList(emails, IdentityType.Email, InputUtil.IdentityInputType.Raw)); } final JsonArray emailHashes = JsonParseUtils.parseArray(obj, "email_hash", rc); if (emailHashes != null && !emailHashes.isEmpty()) { - if (getInputList != null) { + if (getV2Request != null) { return null; // only one type of input is allowed } - getInputList = () -> createInputList(emailHashes, IdentityType.Email, InputUtil.IdentityInputType.Hash); + getV2Request = () -> new IdentityMapV2Input("email_hash", createInputList(emailHashes, IdentityType.Email, InputUtil.IdentityInputType.Hash)); } final JsonArray phones = this.phoneSupport ? JsonParseUtils.parseArray(obj,"phone", rc) : null; if (phones != null && !phones.isEmpty()) { - if (getInputList != null) { + if (getV2Request != null) { return null; // only one type of input is allowed } - getInputList = () -> createInputList(phones, IdentityType.Phone, InputUtil.IdentityInputType.Raw); + getV2Request = () -> new IdentityMapV2Input("phone", createInputList(phones, IdentityType.Phone, InputUtil.IdentityInputType.Raw)); } final JsonArray phoneHashes = this.phoneSupport ? JsonParseUtils.parseArray(obj,"phone_hash", rc) : null; if (phoneHashes != null && !phoneHashes.isEmpty()) { - if (getInputList != null) { + if (getV2Request != null) { return null; // only one type of input is allowed } - getInputList = () -> createInputList(phoneHashes, IdentityType.Phone, InputUtil.IdentityInputType.Hash); + getV2Request = () -> new IdentityMapV2Input("phone_hash", createInputList(phoneHashes, IdentityType.Phone, InputUtil.IdentityInputType.Hash)); } if (emails == null && emailHashes == null && phones == null && phoneHashes == null) { return null; } - return getInputList == null ? - createInputList(null, IdentityType.Email, InputUtil.IdentityInputType.Raw) : // handle empty array - getInputList.get(); + return getV2Request == null ? + new IdentityMapV2Input("email", createInputList(null, IdentityType.Email, InputUtil.IdentityInputType.Raw)) : // handle empty array + getV2Request.get(); } private void handleIdentityMapV3(RoutingContext rc) { @@ -1317,71 +1334,92 @@ private static String getApiContact(RoutingContext rc) { return apiContact; } - private void recordIdentityMapStats(RoutingContext rc, int inputCount, int invalidCount, int optoutCount) { + private void recordIdentityMapStats(RoutingContext rc, Map diiTypeCounts, int invalidCount, int optoutCount) { String apiContact = getApiContact(rc); + String path = rc.request().path(); + + for (Map.Entry entry : diiTypeCounts.entrySet()) { + String diiType = entry.getKey(); + int count = entry.getValue(); + if (count > 0) { + String cacheKey = apiContact + "|" + path + "|" + diiType; + DistributionSummary ds = _identityMapMetricSummaries.computeIfAbsent(cacheKey, k -> DistributionSummary + .builder("uid2_operator_identity_map_inputs") + .description("number of identifiers passed to identity map endpoint") + .tags("api_contact", apiContact, "path", path, "dii_type", diiType) + .register(Metrics.globalRegistry)); + ds.record(count); + } + } - DistributionSummary ds = _identityMapMetricSummaries.computeIfAbsent(apiContact, k -> DistributionSummary - .builder("uid2_operator_identity_map_inputs") - .description("number of emails or email hashes passed to identity map batch endpoint") - .tags("api_contact", apiContact) - .register(Metrics.globalRegistry)); - ds.record(inputCount); - - Tuple.Tuple2 ids = _identityMapUnmappedIdentifiers.computeIfAbsent(apiContact, k -> new Tuple.Tuple2<>( + String cacheKey = apiContact + "|" + path; + Tuple.Tuple2 ids = _identityMapUnmappedIdentifiers.computeIfAbsent(cacheKey, k -> new Tuple.Tuple2<>( Counter.builder("uid2_operator_identity_map_unmapped_total") .description("invalid identifiers") - .tags("api_contact", apiContact, "reason", "invalid") + .tags("api_contact", apiContact, "reason", "invalid", "path", path) .register(Metrics.globalRegistry), Counter.builder("uid2_operator_identity_map_unmapped_total") .description("optout identifiers") - .tags("api_contact", apiContact, "reason", "optout") + .tags("api_contact", apiContact, "reason", "optout", "path", path) .register(Metrics.globalRegistry))); if (invalidCount > 0) ids.getItem1().increment(invalidCount); if (optoutCount > 0) ids.getItem2().increment(optoutCount); - Counter rs = _identityMapRequestWithUnmapped.computeIfAbsent(apiContact, k -> Counter + Counter rs = _identityMapRequestWithUnmapped.computeIfAbsent(cacheKey, k -> Counter .builder("uid2_operator_identity_map_unmapped_requests_total") .description("number of requests with unmapped identifiers") - .tags("api_contact", apiContact) + .tags("api_contact", apiContact, "path", path) .register(Metrics.globalRegistry)); if (invalidCount > 0 || optoutCount > 0) { rs.increment(); } - recordIdentityMapStatsForServiceLinks(rc, apiContact, inputCount, invalidCount, optoutCount); + recordIdentityMapStatsForServiceLinks(rc, apiContact, path, diiTypeCounts, invalidCount, optoutCount); } - private void recordIdentityMapStatsForServiceLinks(RoutingContext rc, String apiContact, int inputCount, - int invalidCount, int optOutCount) { + private void recordIdentityMapStatsForServiceLinks(RoutingContext rc, String apiContact, String path, + Map diiTypeCounts, int invalidCount, int optOutCount) { // If request is from a service, break it down further by link_id String serviceLinkName = rc.get(SecureLinkValidatorService.SERVICE_LINK_NAME, ""); if (!serviceLinkName.isBlank()) { // serviceName will be non-empty as it will be inserted during validation final String serviceName = rc.get(SecureLinkValidatorService.SERVICE_NAME); - final String metricKey = serviceName + serviceLinkName; - DistributionSummary ds = _identityMapMetricSummaries.computeIfAbsent(metricKey, - k -> DistributionSummary.builder("uid2_operator_identity_map_services_inputs") - .description("number of emails or phone numbers passed to identity map batch endpoint by services") - .tags(Arrays.asList(Tag.of("api_contact", apiContact), - Tag.of("service_name", serviceName), - Tag.of("service_link_name", serviceLinkName))) - .register(Metrics.globalRegistry)); - ds.record(inputCount); - - Tuple.Tuple2 counterTuple = _identityMapUnmappedIdentifiers.computeIfAbsent(metricKey, + + for (Map.Entry entry : diiTypeCounts.entrySet()) { + String diiType = entry.getKey(); + int count = entry.getValue(); + if (count > 0) { + final String cacheKey = serviceName + serviceLinkName + "|" + path + "|" + diiType; + DistributionSummary ds = _identityMapServicesMetricSummaries.computeIfAbsent(cacheKey, + k -> DistributionSummary.builder("uid2_operator_identity_map_services_inputs") + .description("number of identifiers passed to identity map endpoint by services") + .tags(Arrays.asList(Tag.of("api_contact", apiContact), + Tag.of("service_name", serviceName), + Tag.of("service_link_name", serviceLinkName), + Tag.of("path", path), + Tag.of("dii_type", diiType))) + .register(Metrics.globalRegistry)); + ds.record(count); + } + } + + final String cacheKey = serviceName + serviceLinkName + "|" + path; + Tuple.Tuple2 counterTuple = _identityMapServicesUnmappedIdentifiers.computeIfAbsent(cacheKey, k -> new Tuple.Tuple2<>( Counter.builder("uid2_operator_identity_map_services_unmapped_total") .description("number of invalid identifiers passed to identity map batch endpoint by services") .tags(Arrays.asList(Tag.of("api_contact", apiContact), Tag.of("reason", "invalid"), Tag.of("service_name", serviceName), - Tag.of("service_link_name", serviceLinkName))) + Tag.of("service_link_name", serviceLinkName), + Tag.of("path", path))) .register(Metrics.globalRegistry), Counter.builder("uid2_operator_identity_map_services_unmapped_total") .description("number of optout identifiers passed to identity map batch endpoint by services") .tags(Arrays.asList(Tag.of("api_contact", apiContact), Tag.of("reason", "optout"), Tag.of("service_name", serviceName), - Tag.of("service_link_name", serviceLinkName))) + Tag.of("service_link_name", serviceLinkName), + Tag.of("path", path))) .register(Metrics.globalRegistry))); if (invalidCount > 0) counterTuple.getItem1().increment(invalidCount); if (optOutCount > 0) counterTuple.getItem2().increment(optOutCount); @@ -1441,14 +1479,14 @@ private void handleOptoutStatus(RoutingContext rc) { private void recordOptOutStatusEndpointStats(RoutingContext rc, int inputCount, int optOutCount) { String apiContact = getApiContact(rc); - DistributionSummary inputDistSummary = optOutStatusCounters.computeIfAbsent(apiContact, k -> DistributionSummary + DistributionSummary inputDistSummary = optOutStatusInputSizeCounters.computeIfAbsent(apiContact, k -> DistributionSummary .builder("uid2_operator_optout_status_input_size") .description("number of UIDs received in request") .tags("api_contact", apiContact) .register(Metrics.globalRegistry)); inputDistSummary.record(inputCount); - DistributionSummary optOutDistSummary = optOutStatusCounters.computeIfAbsent(apiContact, k -> DistributionSummary + DistributionSummary optOutDistSummary = optOutStatusOptOutSizeCounters.computeIfAbsent(apiContact, k -> DistributionSummary .builder("uid2_operator_optout_status_optout_size") .description("number of UIDs that have opted out") .tags("api_contact", apiContact) diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index a91bc57bb..7159adc9d 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -32,6 +32,7 @@ import com.uid2.shared.store.*; import com.uid2.shared.store.reader.RotatingKeysetProvider; import com.uid2.shared.store.salt.ISaltProvider; +import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.vertx.core.AsyncResult; @@ -2904,6 +2905,72 @@ void identityMapBatchRequestTooLargeForPhone(Vertx vertx, VertxTestContext testC send(vertx, "v2/identity/map", req, 413, json -> testContext.completeNow()); } + private double getIdentityMapInputAmount(String path, String diiType) { + DistributionSummary ds = registry + .find("uid2_operator_identity_map_inputs") + .tag("path", path) + .tag("dii_type", diiType) + .summary(); + return ds == null ? 0.0 : ds.totalAmount(); + } + + @Test + void identityMapV2InputMetricTaggedWithEmailDiiType(Vertx vertx, VertxTestContext testContext) { + fakeAuth(201, Role.MAPPER); + setupKeys(); + setupSalts(); + + JsonObject req = new JsonObject() + .put("email", new JsonArray().add("test1@uid2.com").add("test2@uid2.com")); + + send(vertx, "v2/identity/map", req, 200, json -> { + assertEquals(2.0, getIdentityMapInputAmount("/v2/identity/map", "email")); + assertEquals(0.0, getIdentityMapInputAmount("/v2/identity/map", "email_hash")); + assertEquals(0.0, getIdentityMapInputAmount("/v2/identity/map", "phone")); + assertEquals(0.0, getIdentityMapInputAmount("/v2/identity/map", "phone_hash")); + testContext.completeNow(); + }); + } + + @Test + void identityMapV2InputMetricTaggedWithEmailHashDiiType(Vertx vertx, VertxTestContext testContext) { + fakeAuth(201, Role.MAPPER); + setupKeys(); + setupSalts(); + + JsonObject req = new JsonObject() + .put("email_hash", new JsonArray() + .add(TokenUtils.getIdentityHashString("test1@uid2.com"))); + + send(vertx, "v2/identity/map", req, 200, json -> { + assertEquals(0.0, getIdentityMapInputAmount("/v2/identity/map", "email")); + assertEquals(1.0, getIdentityMapInputAmount("/v2/identity/map", "email_hash")); + assertEquals(0.0, getIdentityMapInputAmount("/v2/identity/map", "phone")); + assertEquals(0.0, getIdentityMapInputAmount("/v2/identity/map", "phone_hash")); + testContext.completeNow(); + }); + } + + @Test + void identityMapV3InputMetricTaggedWithPathAndDiiType(Vertx vertx, VertxTestContext testContext) { + fakeAuth(201, Role.MAPPER); + SaltEntry salt = setupSalts(); + when(saltProviderSnapshot.getRotatingSalt(any())).thenReturn(salt); + + JsonObject req = new JsonObject() + .put("email", new JsonArray().add("test1@uid2.com").add("test2@uid2.com")) + .put("phone_hash", new JsonArray() + .add(TokenUtils.getIdentityHashString("+15555555555"))); + + send(vertx, "v3/identity/map", req, 200, json -> { + assertEquals(2.0, getIdentityMapInputAmount("/v3/identity/map", "email")); + assertEquals(0.0, getIdentityMapInputAmount("/v3/identity/map", "email_hash")); + assertEquals(0.0, getIdentityMapInputAmount("/v3/identity/map", "phone")); + assertEquals(1.0, getIdentityMapInputAmount("/v3/identity/map", "phone_hash")); + testContext.completeNow(); + }); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) void identityMapOptoutDefaultOption(boolean useV4Uid, Vertx vertx, VertxTestContext testContext) {