Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
package eu.bbmri_eric.quality.agent.dataquality;

import eu.bbmri_eric.quality.agent.dataquality.dto.DatabaseHealthDTO;
import eu.bbmri_eric.quality.agent.dataquality.dto.ResultDTO;
import org.json.JSONObject;

/** Provides connectivity to a database on which the Data Quality Checks should be executed. */
public interface DataStore {
/**
* Retrieves an entity of the specified type and ID from the data store.
Expand All @@ -14,4 +17,20 @@ public interface DataStore {
JSONObject getEntity(String entityType, String id) throws Exception;

JSONObject checkHealth() throws Exception;

/**
* Executes a query against the connected database.
*
* @param query the query string to execute. e.g., SQL or CQL
* @return the result of the query execution as a {@link ResultDTO}
*/
ResultDTO executeQuery(String query);

/**
* Checks the health of the database connection.
*
* @return a {@link eu.bbmri_eric.quality.agent.dataquality.dto.DatabaseHealthDTO} containing the
* current health status and metrics of the database connection
*/
DatabaseHealthDTO checkHealthV2();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package eu.bbmri_eric.quality.agent.dataquality;

/** Factory for selecting the active {@link DataStore} based on current settings. */
public interface DataStoreFactory {
/**
* Resolves the data store that should be used based on current settings.
*
* @return the resolved {@link DataStore}
*/
DataStore resolveDataStore();
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
* FHIR resources such as Libraries and Measures. It provides methods to interact with and query the
* FHIR backend.
*/
public interface FHIRStore {
public interface FHIRServer extends DataStore {

/**
* Returns a base JSON template for creating a FHIR Library resource.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package eu.bbmri_eric.quality.agent.dataquality.controller;

import eu.bbmri_eric.quality.agent.dataquality.DataStore;
import eu.bbmri_eric.quality.agent.dataquality.DataStoreFactory;
import java.util.NoSuchElementException;
import org.json.JSONObject;
import org.springframework.http.MediaType;
Expand All @@ -13,15 +14,16 @@
@RestController
@RequestMapping("/api/entities")
class EntityController {
private final DataStore dataStore;
private final DataStoreFactory dataStoreFactory;

EntityController(DataStore dataStore) {
this.dataStore = dataStore;
EntityController(DataStoreFactory dataStoreFactory) {
this.dataStoreFactory = dataStoreFactory;
}

@GetMapping("{entityType}/{id}")
public ResponseEntity<String> getEntity(
@PathVariable String entityType, @PathVariable String id) {
DataStore dataStore = dataStoreFactory.resolveDataStore();
try {
JSONObject response = dataStore.getEntity(entityType, id);
if (response == null) {
Expand All @@ -39,6 +41,7 @@ public ResponseEntity<String> getEntity(

@GetMapping("health")
public ResponseEntity<String> checkHealth() {
DataStore dataStore = dataStoreFactory.resolveDataStore();
try {
JSONObject healthResult = dataStore.checkHealth();

Expand Down
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
package eu.bbmri_eric.quality.agent.dataquality.domain;

import eu.bbmri_eric.quality.agent.dataquality.FHIRStore;
import eu.bbmri_eric.quality.agent.dataquality.DataStore;
import eu.bbmri_eric.quality.agent.dataquality.dto.ResultDTO;

/**
* Represents a singular executable query that determines a specific aspect of data quality.
* Implementations of this interface encapsulate the logic for evaluating data quality against a
* given FHIR data store.
* given data store.
*/
public interface DataQualityCheck {

/**
* Executes the data quality check against the specified FHIR store.
* Executes the data quality check against the specified data store.
*
* @param fhirStore the FHIR store against which the data quality check should be executed
* @param dataStore the data store against which the data quality check should be executed
* @return the result of the data quality check
*/
ResultDTO execute(FHIRStore fhirStore);
ResultDTO execute(DataStore dataStore);

/**
* Returns the human-readable name of the data quality check.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package eu.bbmri_eric.quality.agent.dataquality.domain;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.bbmri_eric.quality.agent.dataquality.FHIRStore;
import eu.bbmri_eric.quality.agent.dataquality.DataStore;
import eu.bbmri_eric.quality.agent.dataquality.dto.ResultDTO;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
Expand All @@ -13,12 +11,8 @@
import jakarta.persistence.Id;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import java.util.Base64;
import java.util.HashSet;
import java.util.Set;
import lombok.Getter;
import lombok.Setter;
import org.json.JSONObject;

/** A data quality check utilizing the Hl7 Clinical Quality Language queries for evaluation. */
@Entity(name = "quality_check")
Expand Down Expand Up @@ -59,42 +53,13 @@ public QualityCheck(String name, String description, String query) {
this.query = query;
}

private static final ObjectMapper mapper = new ObjectMapper();

@Override
public ResultDTO execute(FHIRStore fhirStore) {
public ResultDTO execute(DataStore dataStore) {
try {
String cqlData = Base64.getEncoder().encodeToString(query.getBytes());
String libraryUri = java.util.UUID.randomUUID().toString().toLowerCase();
String measureUri = java.util.UUID.randomUUID().toString().toLowerCase();
JSONObject libraryResource = fhirStore.createLibrary(libraryUri, cqlData);
fhirStore.postResource("Library", libraryResource);
JSONObject measureResource = fhirStore.createMeasure(measureUri, libraryUri, "Patient");
JSONObject measureResponse = fhirStore.postResource("Measure", measureResource);
String measureId = measureResponse.getString("id");
JSONObject measureReport = fhirStore.evaluateMeasureList(measureId);
JsonNode mr = mapper.readTree(measureReport.toString());

int count = mr.at("/group/0/population/0/count").asInt();
Set<String> idSet = new HashSet<>();
if (count != 0) {
String listRef = mr.at("/group/0/population/0/subjectResults/reference").asText(null);
if (listRef != null && listRef.startsWith("List/")) {
String listId = listRef.substring("List/".length());

JSONObject listResource = fhirStore.getPatientList(listId);
JsonNode lr = mapper.readTree(listResource.toString());

for (JsonNode entry : lr.withArray("entry")) {
String ref = entry.at("/item/reference").asText(null);
if (ref != null && ref.startsWith("Patient/")) {
idSet.add(ref.substring("Patient/".length()));
}
}
}
}

return new ResultDTO(count, "Patient", idSet);
ResultDTO result = dataStore.executeQuery(query);
return result != null
? result
: new ResultDTO("No result returned from data store execution.");
} catch (Exception | NoSuchMethodError e) {
Comment thread
RadovanTomik marked this conversation as resolved.
return new ResultDTO(e.getMessage());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,6 @@ public enum QualityCheckType {
CQL,

/** Java-based built-in check implementation. */
JAVA
JAVA,
SQL
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package eu.bbmri_eric.quality.agent.dataquality.dto;

public enum DBStatus {
UP,
DOWN
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package eu.bbmri_eric.quality.agent.dataquality.dto;

import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;

@Getter
@Setter
@NoArgsConstructor(access = AccessLevel.PROTECTED)
@AllArgsConstructor
@Schema(name = "Database Health", description = "Health status information for the database")
public class DatabaseHealthDTO {
@Schema(
description = "Overall health status of the database",
example = "UP",
requiredMode = Schema.RequiredMode.REQUIRED)
public DBStatus status;

@Schema(
description = "Error message if the database is unhealthy",
example = "Connection refused",
requiredMode = Schema.RequiredMode.NOT_REQUIRED)
public String error;

@Schema(
description = "Additional details about the health status",
example = "Database connection pool exhausted",
requiredMode = Schema.RequiredMode.NOT_REQUIRED)
public String details;
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package eu.bbmri_eric.quality.agent.dataquality.impl;

import eu.bbmri_eric.quality.agent.dataquality.FHIRStore;
import eu.bbmri_eric.quality.agent.dataquality.DataStore;
import eu.bbmri_eric.quality.agent.dataquality.FHIRServer;
import eu.bbmri_eric.quality.agent.dataquality.domain.DataQualityCheck;
import eu.bbmri_eric.quality.agent.dataquality.domain.QualityCheck;
import eu.bbmri_eric.quality.agent.dataquality.dto.ResultDTO;
Expand Down Expand Up @@ -33,7 +34,10 @@ class DuplicateIdentifierCheck implements DataQualityCheck {
}

@Override
public ResultDTO execute(FHIRStore fhirStore) {
public ResultDTO execute(DataStore dataStore) {
if (!(dataStore instanceof FHIRServer fhirStore)) {
return new ResultDTO("FHIR data store required for " + getName());
}
try {
List<Resource> patients = fhirStore.fetchAllResources("Patient", List.of("id", "identifier"));
Map<String, List<String>> identifierMap = new HashMap<>();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package eu.bbmri_eric.quality.agent.dataquality.impl;

import eu.bbmri_eric.quality.agent.dataquality.FHIRStore;
import eu.bbmri_eric.quality.agent.dataquality.DataStore;
import eu.bbmri_eric.quality.agent.dataquality.DataStoreFactory;
import eu.bbmri_eric.quality.agent.dataquality.FHIRServer;
import eu.bbmri_eric.quality.agent.dataquality.ReportPipelineStep;
import eu.bbmri_eric.quality.agent.dataquality.domain.Report;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -10,16 +12,22 @@
@Component
class EntityCountStep implements ReportPipelineStep {

private final FHIRStore fhirStore;
private final DataStoreFactory dataStoreFactory;

EntityCountStep(FHIRStore fhirStore) {
this.fhirStore = fhirStore;
EntityCountStep(DataStoreFactory dataStoreFactory) {
this.dataStoreFactory = dataStoreFactory;
}

@Override
public Report execute(Report report) {
log.info("Counting entities for report id: {}", report.getId());

DataStore dataStore = dataStoreFactory.resolveDataStore();
if (!(dataStore instanceof FHIRServer fhirStore)) {
log.info("Skipping entity count: not a FHIR data store");
return report;
}

Integer patientCount = fhirStore.countResources("Patient");
Integer sampleCount = fhirStore.countResources("Specimen");

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package eu.bbmri_eric.quality.agent.dataquality.impl;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.bbmri_eric.quality.agent.dataquality.FHIRServer;
import eu.bbmri_eric.quality.agent.dataquality.dto.ResultDTO;
import java.util.Base64;
import java.util.HashSet;
import java.util.Set;
import org.json.JSONObject;

public final class FhirCqlQueryExecutor {
private static final ObjectMapper mapper = new ObjectMapper();

private FhirCqlQueryExecutor() {}

public static ResultDTO execute(FHIRServer fhirStore, String query) {
try {
String cqlData = Base64.getEncoder().encodeToString(query.getBytes());
String libraryUri = java.util.UUID.randomUUID().toString().toLowerCase();
String measureUri = java.util.UUID.randomUUID().toString().toLowerCase();
JSONObject libraryResource = fhirStore.createLibrary(libraryUri, cqlData);
fhirStore.postResource("Library", libraryResource);
JSONObject measureResource = fhirStore.createMeasure(measureUri, libraryUri, "Patient");
JSONObject measureResponse = fhirStore.postResource("Measure", measureResource);
String measureId = measureResponse.getString("id");
JSONObject measureReport = fhirStore.evaluateMeasureList(measureId);
JsonNode mr = mapper.readTree(measureReport.toString());

int count = mr.at("/group/0/population/0/count").asInt();
Set<String> idSet = new HashSet<>();
if (count != 0) {
String listRef = mr.at("/group/0/population/0/subjectResults/reference").asText(null);
if (listRef != null && listRef.startsWith("List/")) {
String listId = listRef.substring("List/".length());

JSONObject listResource = fhirStore.getPatientList(listId);
JsonNode lr = mapper.readTree(listResource.toString());

for (JsonNode entry : lr.withArray("entry")) {
String ref = entry.at("/item/reference").asText(null);
if (ref != null && ref.startsWith("Patient/")) {
idSet.add(ref.substring("Patient/".length()));
}
}
}
}

return new ResultDTO(count, "Patient", idSet);
} catch (Exception | NoSuchMethodError e) {
Comment thread
RadovanTomik marked this conversation as resolved.
return new ResultDTO(e.getMessage());
}
}
}

This file was deleted.

Loading
Loading