diff --git a/Dockerfile b/Dockerfile index 5aa0ffbbd..5cd9d3e8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ COPY . /usr/src/app WORKDIR /usr/src/app RUN mvn clean install -DskipTests=true -FROM adoptopenjdk/openjdk11:alpine +FROM aehrc/jre:openjdk-17 LABEL maintainer="SNOMED International " ARG SUID=1042 diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..aaec31a98 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,48 @@ +name: rvf-$(Date:yyyyMMdd)$(Rev:.r) + +trigger: + branches: + include: + - 'groups-api-fix' + - 'support-stored-proc-file' +pool: + vmImage: 'ubuntu-latest' + +variables: + mavenCache: $(Pipeline.Workspace)/.m2/repository + mavenOptions: '-Dmaven.repo.local=$(mavenCache)' + +stages: + - stage: build + displayName: Build + jobs: + - job: build + displayName: Build + steps: + - task: Cache@2 + displayName: Cache Maven local repo + inputs: + key: 'maven | "$(Agent.OS)" | **/pom.xml' + restoreKeys: | + maven | "$(Agent.OS)" + maven + path: $(mavenCache) + - task: MavenAuthenticate@0 + inputs: + artifactsFeeds: 'mavenbuild' + - task: Docker@2 + inputs: + containerRegistry: $(acr_registry) + command: 'login' + - task: Maven@3 + displayName: Build and verify + inputs: + mavenPomFile: 'pom.xml' + options: '-Ddocker.image.prefix=$(acr_repo)' + mavenOptions: '$(mavenOptions)' + javaHomeOption: 'JDKVersion' + jdkVersionOption: '1.17' + jdkArchitectureOption: 'x64' + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + goals: 'clean install dockerfile:build dockerfile:push' diff --git a/docker-compose.yml b/docker-compose.yml index 5e96e8fd6..b4c2104ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ volumes: services: db: - image: mysql:5.7 + image: mysql:8.0.28 restart: always environment: - MYSQL_ROOT_PASSWORD=snomed @@ -21,7 +21,7 @@ services: volumes: - mysql:/var/lib/mysql command: - mysqld --sql_mode="NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES" --lower_case_table_names=1 + mysqld --local-infile=ON --sql_mode="NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES" --lower_case_table_names=1 rvf: image: snomedinternational/release-validation-framework:latest container_name: rvf @@ -31,7 +31,7 @@ services: ports: - 8081:8081 environment: - - SPRING_DATASOURCE_URL=jdbc:mysql://db:3306/?useSSL=false + - SPRING_DATASOURCE_URL=jdbc:mysql://db:3306/?useSSL=false&allowLoadLocalInfile=true&allowPublicKeyRetrieval=true - SPRING_DATASOURCE_USERNAME=root - SPRING_DATASOURCE_PASSWORD=snomed - rvf.assertion.resource.local.path=./snomed-release-validation-assertions/ diff --git a/pom.xml b/pom.xml index d9e4f2d1a..6aa1164c0 100644 --- a/pom.xml +++ b/pom.xml @@ -351,6 +351,18 @@ + + + org.owasp + dependency-check-maven + 8.3.1 + + + none + + + + diff --git a/src/main/java/org/ihtsdo/rvf/core/data/model/ExecutionCommand.java b/src/main/java/org/ihtsdo/rvf/core/data/model/ExecutionCommand.java index 9b4f3a835..a18b06ba4 100644 --- a/src/main/java/org/ihtsdo/rvf/core/data/model/ExecutionCommand.java +++ b/src/main/java/org/ihtsdo/rvf/core/data/model/ExecutionCommand.java @@ -18,14 +18,14 @@ public class ExecutionCommand { @GeneratedValue Long id; - @Column(columnDefinition = "text") + @Column(columnDefinition = "longtext") String template = null; @JsonBackReference @OneToOne(fetch = FetchType.EAGER, mappedBy = "command") Test test; @ElementCollection(fetch = FetchType.EAGER) @CollectionTable(name="command_statements", joinColumns=@JoinColumn(name="command_id")) - @Column(name="statement", columnDefinition = "text") + @Column(name="statement", columnDefinition = "longtext") @OrderColumn(name="statement_index") List statements = new ArrayList<>(); diff --git a/src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java b/src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java index dd57cd051..721b5cef3 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java @@ -1,8 +1,22 @@ package org.ihtsdo.rvf.core.service; +import java.sql.*; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import javax.annotation.Resource; +import javax.naming.ConfigurationException; +import javax.naming.ConfigurationException; import org.apache.commons.dbcp.BasicDataSource; import org.ihtsdo.rvf.core.data.model.*; import org.ihtsdo.rvf.core.service.config.MysqlExecutionConfig; +import org.ihtsdo.rvf.core.service.util.MySqlQueryTransformer; import org.ihtsdo.rvf.importer.AssertionGroupImporter.ProductName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -208,7 +222,7 @@ private void executeCommand(final Assertion assertion, final MysqlExecutionConfi else { if (sqlStatement.startsWith("create table")){ // only add engine if we do not create using a like statement - if (!(sqlStatement.contains("like") || sqlStatement.contains("as"))) { + if (!sqlStatement.toUpperCase().contains(" ENGINE") && !(sqlStatement.contains("like") || sqlStatement.contains("as"))) { sqlStatement = sqlStatement + " ENGINE = MyISAM"; } } @@ -218,59 +232,10 @@ private void executeCommand(final Assertion assertion, final MysqlExecutionConfi } private List transformSql(String[] parts, Assertion assertion, MysqlExecutionConfig config) throws ConfigurationException { - List result = new ArrayList<>(); - String defaultCatalog = dataSource.getDefaultCatalog(); - String prospectiveSchema = config.getProspectiveVersion(); - final String[] nameParts = config.getProspectiveVersion().split("_"); - String defaultModuleId = StringUtils.hasLength(config.getDefaultModuleId()) ? config.getDefaultModuleId() : (nameParts.length >= 2 ? ProductName.toModuleId(nameParts[1]) : "NOT_SUPPLIED"); - String includedModules = config.getIncludedModules().stream().collect(Collectors.joining(",")); - String version = (nameParts.length >= 3 ? nameParts[2] : "NOT_SUPPLIED"); - - String previousReleaseSchema = config.getPreviousVersion(); - String dependencyReleaseSchema = config.getExtensionDependencyVersion(); - - //We need both these schemas to exist - if (prospectiveSchema == null) { - throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + prospectiveSchema); - } - - if (config.isReleaseValidation() && !config.isFirstTimeRelease() && previousReleaseSchema == null) { - throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + previousReleaseSchema); - } - for( String part : parts) { - if ((part.contains("") && previousReleaseSchema == null) - || (part.contains("") && dependencyReleaseSchema == null)) { - continue; - } - - logger.debug("Original sql statement: {}", part); - // remove all SQL comments - //TODO might throw errors for -- style comments - final Pattern commentPattern = Pattern.compile("/\\*.*?\\*/", Pattern.DOTALL); - part = commentPattern.matcher(part).replaceAll(""); - // replace all substitutions for exec - part = part.replaceAll("", String.valueOf(config.getExecutionId())); - part = part.replaceAll("", String.valueOf(assertion.getAssertionId())); - part = part.replaceAll("", defaultModuleId); - part = part.replaceAll("", includedModules); - part = part.replaceAll("", version); - // watch out for any 's that users might have introduced - part = part.replaceAll("qa_result", defaultCatalog+ "." + qaResulTableName); - part = part.replaceAll("", prospectiveSchema); - part = part.replaceAll("", prospectiveSchema); - if (previousReleaseSchema != null) { - part = part.replaceAll("", previousReleaseSchema); - } - if (dependencyReleaseSchema != null) { - part = part.replaceAll("", dependencyReleaseSchema); - } - part = part.replaceAll("", deltaTableSuffix); - part = part.replaceAll("", snapshotTableSuffix); - part = part.replaceAll("", fullTableSuffix); - part.trim(); - logger.debug("Transformed sql statement: {}", part); - result.add(part); - } - return result; + String qaResult = dataSource.getDefaultCatalog()+ "." + qaResulTableName; + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + Map configMap = Map.of("qa_result", qaResult, "", String.valueOf(assertion.getAssertionId())); + return queryTransformer.transformSql(parts, config, configMap); } diff --git a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java new file mode 100644 index 000000000..e2a874367 --- /dev/null +++ b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java @@ -0,0 +1,129 @@ +package org.ihtsdo.rvf.core.service.util; + +import com.facebook.presto.sql.parser.StatementSplitter; +import com.google.common.collect.ImmutableSet; +import org.apache.commons.dbcp.BasicDataSource; +import org.ihtsdo.otf.rest.exception.BusinessServiceException; +import org.ihtsdo.rvf.core.data.model.Assertion; +import org.ihtsdo.rvf.core.service.config.MysqlExecutionConfig; +import org.ihtsdo.rvf.importer.AssertionGroupImporter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.StringUtils; + +import javax.naming.ConfigurationException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class MySqlQueryTransformer { + private final Logger logger = LoggerFactory.getLogger(MySqlQueryTransformer.class); + + private static final String FAILED_TO_FIND_RVF_DB_SCHEMA = "Failed to find rvf db schema for "; + + private String deltaTableSuffix = "d"; + private String snapshotTableSuffix = "s"; + private String fullTableSuffix = "f"; + private static final String DEFAULT_DELIMITER = ";"; + private static final String DELIMITER_REGEX_PATTERN = "^[ ]*(delimiter|DELIMITER)"; + + public List transformSql(String[] parts, MysqlExecutionConfig config, final Map configMap) + throws ConfigurationException { + + logger.info("Config Map contains " + configMap.entrySet().stream().map(e -> e.getKey() + " : " + e.getValue()).collect(Collectors.joining(","))); + + List result = new ArrayList<>(); + String prospectiveSchema = config.getProspectiveVersion(); + String previousReleaseSchema = config.getPreviousVersion(); + String dependencyReleaseSchema = config.getExtensionDependencyVersion(); + + //We need both these schemas to exist + if (prospectiveSchema == null) { + throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + prospectiveSchema); + } + + if (config.isReleaseValidation() && !config.isFirstTimeRelease() && previousReleaseSchema == null) { + throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + previousReleaseSchema); + } + + final String[] nameParts = config.getProspectiveVersion().split("_"); + String version = (nameParts.length >= 3 ? nameParts[2] : "NOT_SUPPLIED"); + String includedModules = config.getIncludedModules().stream().collect(Collectors.joining(",")); + String defaultModuleId = StringUtils.hasLength(config.getDefaultModuleId()) ? config.getDefaultModuleId() : (nameParts.length >= 2 ? AssertionGroupImporter.ProductName.toModuleId(nameParts[1]) : "NOT_SUPPLIED"); + for( String part : parts) { + if ((part.contains("") && previousReleaseSchema == null) + || (part.contains("") && dependencyReleaseSchema == null)) { + continue; + } + + logger.debug("Original sql statement: {}", part); + // remove all SQL comments - //TODO might throw errors for -- style comments + final Pattern commentPattern = Pattern.compile("/\\*.*?\\*/", Pattern.DOTALL); + part = commentPattern.matcher(part).replaceAll(""); + // replace all substitutions for exec + part = part.replaceAll("", String.valueOf(config.getExecutionId())); + part = part.replaceAll("", configMap.get("")); + part = part.replaceAll("", defaultModuleId); + part = part.replaceAll("", includedModules); + part = part.replaceAll("", version); + // watch out for any 's that users might have introduced + part = part.replaceAll("qa_result", configMap.get("qa_result")); + part = part.replaceAll("", prospectiveSchema); + part = part.replaceAll("", prospectiveSchema); + if (previousReleaseSchema != null) { + part = part.replaceAll("", previousReleaseSchema); + } + if (dependencyReleaseSchema != null) { + part = part.replaceAll("", dependencyReleaseSchema); + } + part = part.replaceAll("", deltaTableSuffix); + part = part.replaceAll("", snapshotTableSuffix); + part = part.replaceAll("", fullTableSuffix); + part = part.replaceAll(Pattern.quote("[[:<:]]"),"\\\\b" ); + part = part.replaceAll(Pattern.quote("[[:>:]]"),"\\\\b" ); + for(Map.Entry configMapEntry: configMap.entrySet()){ + if (configMapEntry.getKey().matches("^<[^>]+>$")) { + part = part.replaceAll(configMapEntry.getKey(), configMapEntry.getValue()); + } + } + part.trim(); + logger.debug("Transformed sql statement: {}", part); + result.add(part); + } + return result; + } + /** + * Convert given sql file content to multiple statements + * @param sqlFileContent + * @return + */ + public List transformToStatements(String sqlFileContent) throws BusinessServiceException { + String delimiter = DEFAULT_DELIMITER; + List result = new ArrayList<>(); + String[] sqlChunks = sqlFileContent.trim().split(DELIMITER_REGEX_PATTERN, Pattern.MULTILINE); + for (int i = 0; i < sqlChunks.length; i++) { + String sqlChunk = sqlChunks[i].trim(); + if (!sqlChunk.isEmpty()) { + if (i > 0) { + delimiter = sqlChunk.trim().replaceAll("(?s)^([^ \r\n]+).*$", "$1"); + sqlChunk = sqlChunk.trim().replaceAll("(?s)^[^ \r\n]+(.*)$", "$1").trim(); + } + if (!sqlChunk.isEmpty()) { + logger.debug("Executing pre-requisite SQL: " + sqlChunk); + final StatementSplitter splitter = new StatementSplitter(sqlChunk, ImmutableSet.of(delimiter)); + if (splitter.getCompleteStatements() == null || splitter.getCompleteStatements().isEmpty()) { + String errorMsg = String.format("SQL statements not ending with %s %s",delimiter, sqlChunk); + logger.error( errorMsg); + throw new BusinessServiceException(errorMsg); + } + result= splitter.getCompleteStatements().stream().map(s -> s.statement()).collect(Collectors.toList()); + + } + } + + } + return result; + } +} diff --git a/src/main/java/org/ihtsdo/rvf/core/service/util/RF2FileTableMapper.java b/src/main/java/org/ihtsdo/rvf/core/service/util/RF2FileTableMapper.java index 65595c798..4d0661e27 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/util/RF2FileTableMapper.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/util/RF2FileTableMapper.java @@ -1,7 +1,7 @@ package org.ihtsdo.rvf.core.service.util; import java.util.Collection; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.regex.Pattern; @@ -11,6 +11,7 @@ public class RF2FileTableMapper { private static final String STATED_RELATIONSHIP_FILE_HEADER = "sct2_StatedRelationship_"; private static final String SIMPLE_FILE_HEADER = "der2_Refset_.*Simple"; private static final String SIMPLE_MAP_FILE_HEADER = "der2_sRefset_.*SimpleMap"; + private static final String INTEGER_SIMPLE_MAP_FILE_HEADER = "der2_iRefset_.*SimpleMap"; private static final String RELATIONSHIP_FILE_HEADER = "sct2_Relationship_"; private static final String RELATIONSHIP_CONCRETE_VALUES_FILE_HEADER = "sct2_RelationshipConcreteValues"; private static final String LANGUAGE_FILE_HEADER = "der2_cRefset_.*Language"; @@ -30,7 +31,14 @@ public class RF2FileTableMapper { private static final String MRCM_ATRRIBUTE_RANGE_FILE_HEADER = "der2_ssccRefset_.*MRCMAttributeRange"; private static final String MRCM_DOMAIN_FILE_HEADER = "der2_sssssssRefset_.*MRCMDomain"; private static final String DESCRIPTION_TYPE_FILE_HEADER = "der2_ciRefset_.*DescriptionType"; - private static final Map tableNameMap = new HashMap<>(); + private static final String ATTRIBUTE_VALUE_MAP_FILE_HEADER = "der2_csRefset_.*AttributeValueMap"; + private static final String EXTENDED_ASSOCIATION_FILE_HEADER = "der2_ccRefset_.*ExtendedAssociation"; + private static final String IDENTIFIER_FILE_HEADER = "sct2_Identifier_"; + private static final String CCS_REFSET_FILE_HEADER = "der2_ccsRefset_"; + private static final String CCI_REFSET_FILE_HEADER = "der2_cciRefset_"; + private static final String C_REFSET_FILE_HEADER = "der2_cRefset_"; + + private static final Map tableNameMap = new LinkedHashMap<>(); private static final String DELTA = ".*Delta.*_*_\\d{8}.txt"; private static final String SNAPSHOT = ".*Snapshot.*_*_\\d{8}.txt"; private static final String FULL = ".*Full.*_*_\\d{8}.txt"; @@ -60,7 +68,14 @@ public class RF2FileTableMapper { tableNameMap.put(MRCM_ATRRIBUTE_RANGE_FILE_HEADER + DELTA, "mrcmattributerangerefset_d"); tableNameMap.put(MRCM_DOMAIN_FILE_HEADER + DELTA, "mrcmdomainrefset_d"); tableNameMap.put(DESCRIPTION_TYPE_FILE_HEADER + DELTA, "descriptiontyperefset_d"); - + tableNameMap.put(ATTRIBUTE_VALUE_MAP_FILE_HEADER + DELTA, "attributevaluemap_d"); + tableNameMap.put(EXTENDED_ASSOCIATION_FILE_HEADER + DELTA, "extendedassociation_d"); + tableNameMap.put(IDENTIFIER_FILE_HEADER + DELTA, "identifier_d"); + tableNameMap.put(INTEGER_SIMPLE_MAP_FILE_HEADER + DELTA, "isimplemaprefset_d"); + tableNameMap.put(CCS_REFSET_FILE_HEADER + DELTA, "ccsrefset_d"); + tableNameMap.put(CCI_REFSET_FILE_HEADER + DELTA, "ccirefset_d"); + tableNameMap.put(C_REFSET_FILE_HEADER + DELTA, "crefset_d"); + //Full tableNameMap.put(CONCEPT_FILE_HEADER + FULL, "concept_f"); tableNameMap.put(DESCRIPTION_FILE_HEADER + FULL, "description_f"); @@ -85,7 +100,14 @@ public class RF2FileTableMapper { tableNameMap.put(MRCM_ATRRIBUTE_RANGE_FILE_HEADER + FULL, "mrcmattributerangerefset_f"); tableNameMap.put(MRCM_DOMAIN_FILE_HEADER + FULL, "mrcmdomainrefset_f"); tableNameMap.put(DESCRIPTION_TYPE_FILE_HEADER + FULL, "descriptiontyperefset_f"); - + tableNameMap.put(ATTRIBUTE_VALUE_MAP_FILE_HEADER + FULL, "attributevaluemap_f"); + tableNameMap.put(EXTENDED_ASSOCIATION_FILE_HEADER + FULL, "extendedassociation_f"); + tableNameMap.put(IDENTIFIER_FILE_HEADER + FULL, "identifier_f"); + tableNameMap.put(INTEGER_SIMPLE_MAP_FILE_HEADER + FULL, "isimplemaprefset_f"); + tableNameMap.put(CCS_REFSET_FILE_HEADER + FULL, "ccsrefset_f"); + tableNameMap.put(CCI_REFSET_FILE_HEADER + FULL, "ccirefset_f"); + tableNameMap.put(C_REFSET_FILE_HEADER + FULL, "crefset_f"); + //Snapshot tableNameMap.put(CONCEPT_FILE_HEADER + SNAPSHOT, "concept_s"); tableNameMap.put(DESCRIPTION_FILE_HEADER + SNAPSHOT, "description_s"); @@ -110,6 +132,13 @@ public class RF2FileTableMapper { tableNameMap.put(MRCM_ATRRIBUTE_RANGE_FILE_HEADER + SNAPSHOT, "mrcmattributerangerefset_s"); tableNameMap.put(MRCM_DOMAIN_FILE_HEADER + SNAPSHOT, "mrcmdomainrefset_s"); tableNameMap.put(DESCRIPTION_TYPE_FILE_HEADER + SNAPSHOT, "descriptiontyperefset_s"); + tableNameMap.put(ATTRIBUTE_VALUE_MAP_FILE_HEADER + SNAPSHOT, "attributevaluemap_s"); + tableNameMap.put(EXTENDED_ASSOCIATION_FILE_HEADER + SNAPSHOT, "extendedassociation_s"); + tableNameMap.put(IDENTIFIER_FILE_HEADER + SNAPSHOT, "identifier_s"); + tableNameMap.put(INTEGER_SIMPLE_MAP_FILE_HEADER + SNAPSHOT, "isimplemaprefset_s"); + tableNameMap.put(CCS_REFSET_FILE_HEADER + SNAPSHOT, "ccsrefset_s"); + tableNameMap.put(CCI_REFSET_FILE_HEADER + SNAPSHOT, "ccirefset_s"); + tableNameMap.put(C_REFSET_FILE_HEADER + SNAPSHOT, "crefset_s"); } public static String getLegacyTableName(final String filename) { diff --git a/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java b/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java index fbf62e12b..f8b9bd9aa 100644 --- a/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java +++ b/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java @@ -3,11 +3,13 @@ import com.facebook.presto.sql.parser.StatementSplitter; import org.apache.commons.io.IOUtils; import org.ihtsdo.otf.resourcemanager.ResourceManager; +import org.ihtsdo.otf.rest.exception.BusinessServiceException; import org.ihtsdo.rvf.core.data.model.Assertion; import org.ihtsdo.rvf.core.data.model.ExecutionCommand; import org.ihtsdo.rvf.core.data.model.Test; import org.ihtsdo.rvf.core.data.model.TestType; import org.ihtsdo.rvf.core.service.AssertionService; +import org.ihtsdo.rvf.core.service.util.MySqlQueryTransformer; import org.jdom2.Document; import org.jdom2.Element; import org.jdom2.JDOMException; @@ -38,6 +40,7 @@ public class AssertionsDatabaseImporter { private static final String CREATE_PROCEDURE = "CREATE PROCEDURE"; + private static final String CREATE_FUNCTION = "CREATE FUNCTION"; private static final Logger logger = LoggerFactory.getLogger(AssertionsDatabaseImporter.class); private static final String RESOURCE_PATH_SEPARATOR = "/"; @@ -125,7 +128,21 @@ private void addSqlTestsToAssertion(Assertion assertion, Element element, String } final String sqlString = IOUtils.toString(sqlInputStream, UTF_8); // add test to assertion - addSqlTestToAssertion(assertion, sqlString); + if(isPreRequisitesSql(sqlFileName)){ //handle pre-requisites.sql + try { + addPreRequisiteSqlToAssertion(assertion, sqlString); + } catch (Exception e) { + String errorMsg = "Failed to add pre-requisite sql script test to assertion with uuid:" + assertion.getUuid(); + logger.error(errorMsg, e); + throw new IllegalStateException(errorMsg, e); + } + } else{ + addSqlTestToAssertion(assertion, sqlString); + } + } + + private boolean isPreRequisitesSql(String sqlFileName){ + return sqlFileName.equalsIgnoreCase("pre-requisites.sql"); } private Assertion createAssertionFromElement(final Element element){ @@ -153,7 +170,7 @@ public void addSqlTestToAssertion(final Assertion assertion, final String sql){ for (final StatementSplitter.Statement statement : splitter.getCompleteStatements()) { String cleanedSql = statement.statement(); logger.debug("sql to be cleaned:" + cleanedSql); - if ( cleanedSql.startsWith(CREATE_PROCEDURE) || cleanedSql.startsWith(CREATE_PROCEDURE.toLowerCase())) { + if( cleanedSql.toUpperCase().startsWith(CREATE_PROCEDURE) || cleanedSql.toUpperCase().startsWith(CREATE_FUNCTION)) { storedProcedureFound = true; } // Process SQL statement @@ -195,6 +212,12 @@ public void addSqlTestToAssertion(final Assertion assertion, final String sql){ uploadTest(assertion, sql, statements); } + protected void addPreRequisiteSqlToAssertion(final Assertion assertion, String preRequisiteSql) throws BusinessServiceException { + MySqlQueryTransformer mySqlQueryTransformer = new MySqlQueryTransformer(); + final List sqlStatements = mySqlQueryTransformer.transformToStatements(preRequisiteSql); + uploadTest(assertion, preRequisiteSql, sqlStatements); + } + private void uploadTest (Assertion assertion, String originalSql, List sqlStatements) { final ExecutionCommand command = new ExecutionCommand(); command.setTemplate(originalSql); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index d4aa922ea..781154f35 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -92,8 +92,8 @@ spring.mvc.pathmatch.matching-strategy= ANT_PATH_MATCHER spring.security.user.name=user spring.security.user.password=password # REST API max request file size -spring.servlet.multipart.max-file-size=1GB -spring.servlet.multipart.max-request-size=1GB +spring.servlet.multipart.max-file-size=2GB +spring.servlet.multipart.max-request-size=2GB # Handles X-Forwarded-XXX headers # Support additional web proxy headers @@ -136,4 +136,4 @@ spring.cloud.vault.enabled=false #spring.cloud.vault.scheme=http #spring.cloud.vault.kv.enabled=true #spring.config.import=consul:localhost:8500,vault:// -#spring.cloud.vault.uri=http://localhost:8200 \ No newline at end of file +#spring.cloud.vault.uri=http://localhost:8200 diff --git a/src/main/resources/sql/create-resource-tables.sql b/src/main/resources/sql/create-resource-tables.sql index 7f1986fbb..ad802bddd 100644 --- a/src/main/resources/sql/create-resource-tables.sql +++ b/src/main/resources/sql/create-resource-tables.sql @@ -1,7 +1,7 @@ drop table if exists res_us_gb_terms; create table res_us_gb_terms ( - us_term varchar(255), - gb_term varchar(255) + us_term VARCHAR(333), + gb_term VARCHAR(333) )engine=myisam default charset=utf8; create index idx_res_usTermMap on res_us_gb_terms(us_term); create index idx_res_gbTermMap on res_us_gb_terms(gb_term); @@ -10,7 +10,7 @@ create index idx_res_gbTermMap on res_us_gb_terms(gb_term); drop table if exists res_casesensitiveTerm; create table res_casesensitiveTerm( - casesensitiveTerm VARCHAR(255) not null + casesensitiveTerm VARCHAR(333) not null )engine=myisam default charset=utf8; create index idx_casesensitiveTerm on res_casesensitiveTerm(casesensitiveTerm); diff --git a/src/main/resources/sql/create-tables-mysql.sql b/src/main/resources/sql/create-tables-mysql.sql index bbde68735..128643751 100644 --- a/src/main/resources/sql/create-tables-mysql.sql +++ b/src/main/resources/sql/create-tables-mysql.sql @@ -24,7 +24,7 @@ moduleid bigint(20) not null, conceptid bigint(20) not null, languagecode varchar(2) not null, typeid bigint(20) not null, -term varchar(256) not null collate utf8_bin, +term VARCHAR(333) not null collate utf8_bin, casesignificanceid bigint(20) not null, key idx_id(id), key idx_effectivetime(effectivetime), @@ -407,6 +407,143 @@ create table descriptiontyperefset_f( key idx_descriptionlength(descriptionlength) ) engine=myisam default charset=utf8; + +drop table if exists attributevaluemap_f; +create table attributevaluemap_f( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + maptarget varchar(32) not null, + value varchar(32) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_maptarget(maptarget) +) engine=myisam default charset=utf8; + +drop table if exists extendedassociation_f; +create table extendedassociation_f( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + targetcomponentid bigint(20) not null, + value varchar(32) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_targetcomponentid(targetcomponentid), + key idx_value(value) +) engine=myisam default charset=utf8; + +drop table if exists identifier_f; +create table identifier_f( + identifierschemeid bigint(20) not null, + alternateidentifier varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + referencedcomponentid bigint(20) not null, + key idx_identifierschemeid(identifierschemeid), + key idx_alternateidentifier(alternateidentifier), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_referencedcomponentid(referencedcomponentid) +) engine=myisam default charset=utf8; + +drop table if exists isimplemaprefset_f; +create table isimplemaprefset_f( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + maptarget bigint(20) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_maptarget(maptarget) +) engine=myisam default charset=utf8; + + +drop table if exists ccsrefset_f; +create table ccsrefset_f( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + componentid1 bigint(20) not null, + componentid2 bigint(20) not null, + value varchar(32) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_componentid1(componentid1), + key idx_componentid2(componentid2), + key idx_value(value) +) engine=myisam default charset=utf8; + +drop table if exists ccirefset_f; +create table ccirefset_f( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + componentid1 bigint(20) not null, + componentid2 bigint(20) not null, + value bigint(20) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_componentid1(componentid1), + key idx_componentid2(componentid2), + key idx_value(value) +) engine=myisam default charset=utf8; + +drop table if exists crefset_f; +create table crefset_f( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + componentid bigint(20) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_componentid(componentid) +) engine=myisam default charset=utf8; + /* create the Snapshot S-CT data tables */ drop table if exists concept_s; @@ -433,7 +570,7 @@ moduleid bigint(20) not null, conceptid bigint(20) not null, languagecode varchar(2) not null, typeid bigint(20) not null, -term varchar(256) not null collate utf8_bin, +term VARCHAR(333) not null collate utf8_bin, casesignificanceid bigint(20) not null, key idx_id(id), key idx_effectivetime(effectivetime), @@ -455,7 +592,7 @@ moduleid bigint(20) not null, conceptid bigint(20) not null, languagecode varchar(2) not null, typeid bigint(20) not null, -term varchar(2048) not null, +term VARCHAR(4096) not null, casesignificanceid bigint(20) not null, key idx_id(id), key idx_effectivetime(effectivetime), @@ -816,6 +953,143 @@ create table descriptiontyperefset_s( key idx_descriptionlength(descriptionlength) ) engine=myisam default charset=utf8; + +drop table if exists attributevaluemap_s; +create table attributevaluemap_s( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + maptarget varchar(32) not null, + value varchar(32) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_maptarget(maptarget) +) engine=myisam default charset=utf8; + +drop table if exists extendedassociation_s; +create table extendedassociation_s( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + targetcomponentid bigint(20) not null, + value varchar(32) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_targetcomponentid(targetcomponentid), + key idx_value(value) +) engine=myisam default charset=utf8; + +drop table if exists identifier_s; +create table identifier_s( + identifierschemeid bigint(20) not null, + alternateidentifier varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + referencedcomponentid bigint(20) not null, + key idx_identifierschemeid(identifierschemeid), + key idx_alternateidentifier(alternateidentifier), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_referencedcomponentid(referencedcomponentid) +) engine=myisam default charset=utf8; + +drop table if exists isimplemaprefset_s; +create table isimplemaprefset_s( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + maptarget bigint(20) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_maptarget(maptarget) +) engine=myisam default charset=utf8; + + +drop table if exists ccsrefset_s; +create table ccsrefset_s( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + componentid1 bigint(20) not null, + componentid2 bigint(20) not null, + value varchar(32) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_componentid1(componentid1), + key idx_componentid2(componentid2), + key idx_value(value) +) engine=myisam default charset=utf8; + +drop table if exists ccirefset_s; +create table ccirefset_s( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + componentid1 bigint(20) not null, + componentid2 bigint(20) not null, + value bigint(20) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_componentid1(componentid1), + key idx_componentid2(componentid2), + key idx_value(value) +) engine=myisam default charset=utf8; + +drop table if exists crefset_s; +create table crefset_s( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + componentid bigint(20) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_componentid(componentid) +) engine=myisam default charset=utf8; + /* create the Delta S-CT data tables */ drop table if exists concept_d; @@ -842,7 +1116,7 @@ moduleid bigint(20) not null, conceptid bigint(20) not null, languagecode varchar(2) not null, typeid bigint(20) not null, -term varchar(256) not null collate utf8_bin, +term VARCHAR(333) not null collate utf8_bin, casesignificanceid bigint(20) not null, key idx_id(id), key idx_effectivetime(effectivetime), @@ -864,7 +1138,7 @@ moduleid bigint(20) not null, conceptid bigint(20) not null, languagecode varchar(2) not null, typeid bigint(20) not null, -term varchar(2048) not null, +term VARCHAR(4096) not null, casesignificanceid bigint(20) not null, key idx_id(id), key idx_effectivetime(effectivetime), @@ -1435,3 +1709,139 @@ create table descriptiontyperefset_d( key idx_descriptionlength(descriptionlength) ) engine=myisam default charset=utf8; + +drop table if exists attributevaluemap_d; +create table attributevaluemap_d( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + maptarget varchar(32) not null, + value varchar(32) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_maptarget(maptarget) +) engine=myisam default charset=utf8; + +drop table if exists extendedassociation_d; +create table extendedassociation_d( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + targetcomponentid bigint(20) not null, + value varchar(32) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_targetcomponentid(targetcomponentid), + key idx_value(value) +) engine=myisam default charset=utf8; + +drop table if exists identifier_d; +create table identifier_d( + identifierschemeid bigint(20) not null, + alternateidentifier varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + referencedcomponentid bigint(20) not null, + key idx_identifierschemeid(identifierschemeid), + key idx_alternateidentifier(alternateidentifier), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_referencedcomponentid(referencedcomponentid) +) engine=myisam default charset=utf8; + +drop table if exists isimplemaprefset_d; +create table isimplemaprefset_d( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + maptarget bigint(20) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_maptarget(maptarget) +) engine=myisam default charset=utf8; + + +drop table if exists ccsrefset_d; +create table ccsrefset_d( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + componentid1 bigint(20) not null, + componentid2 bigint(20) not null, + value varchar(32) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_componentid1(componentid1), + key idx_componentid2(componentid2), + key idx_value(value) +) engine=myisam default charset=utf8; + +drop table if exists ccirefset_d; +create table ccirefset_d( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + componentid1 bigint(20) not null, + componentid2 bigint(20) not null, + value bigint(20) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_componentid1(componentid1), + key idx_componentid2(componentid2), + key idx_value(value) +) engine=myisam default charset=utf8; + +drop table if exists crefset_d; +create table crefset_d( + id varchar(36) not null, + effectivetime char(8) not null, + active char(1) not null, + moduleid bigint(20) not null, + refsetid bigint(20) not null, + referencedcomponentid bigint(20) not null, + componentid bigint(20) not null, + key idx_id(id), + key idx_effectivetime(effectivetime), + key idx_active(active), + key idx_moduleid(moduleid), + key idx_refsetid(refsetid), + key idx_referencedcomponentid(referencedcomponentid), + key idx_componentid(componentid) +) engine=myisam default charset=utf8; diff --git a/src/test/java/org/ihtsdo/rvf/core/service/RF2FileTableMapperTest.java b/src/test/java/org/ihtsdo/rvf/core/service/RF2FileTableMapperTest.java index f0b3cd48f..e018d99b3 100644 --- a/src/test/java/org/ihtsdo/rvf/core/service/RF2FileTableMapperTest.java +++ b/src/test/java/org/ihtsdo/rvf/core/service/RF2FileTableMapperTest.java @@ -108,8 +108,8 @@ public class RF2FileTableMapperTest { "associationrefset_d", "relationship_d" }; - - @Test + + @Test public void testInternationalDeltaFiles() { int i = 0; assertEquals(INT_DELTA_FILES.length, INT_EXPECTED_DELTA.length); diff --git a/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java b/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java new file mode 100644 index 000000000..0db9ecf96 --- /dev/null +++ b/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java @@ -0,0 +1,170 @@ +package org.ihtsdo.rvf.core.service.util; + +import org.ihtsdo.otf.rest.exception.BusinessServiceException; +import org.ihtsdo.rvf.core.service.config.MysqlExecutionConfig; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import javax.naming.ConfigurationException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.when; + + +@ExtendWith(MockitoExtension.class) +public class MySqlQueryTransformerTest { + @Mock + MysqlExecutionConfig config; + @Test + public void transformToStatements() throws BusinessServiceException { + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + String sqlToTest = "DELIMITER //\n" + + "SET default_storage_engine=MYISAM//\n" + + "DROP TABLE IF EXISTS concept_active//\n" + + "CREATE TABLE concept_active ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AS\n" + + "SELECT * FROM concept_ AS concepts WHERE active = 1//\n" + + "create unique index concept_active_id_ix on concept_active(id)//\n" + + "create index concept_active_effectivetime_ix on concept_active(effectivetime)//\n" + + "create index concept_active_definitionstatusid_ix on concept_active(definitionstatusid)//\n" + + "create index concept_active_moduleid_ix on concept_active(moduleid)//\n" + + "create index concept_active_active_ix on concept_active(active)//"; + List result = queryTransformer.transformToStatements(sqlToTest); + assertEquals(8, result.size()); + assertEquals("create index concept_active_active_ix on concept_active(active)", result.get(7)); + assertEquals("CREATE TABLE concept_active ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AS\n" + + "SELECT * FROM concept_ AS concepts WHERE active = 1", result.get(2)); + } + @Test + public void transformToStatementsWithSqlFunctions() throws BusinessServiceException { + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + String sqlToTest = "DELIMITER // drop function if exists get_cr_ADRS_PT//\n" + + "create function get_cr_ADRS_PT(candidate bigint) returns varchar(4500)\n" + + "BEGIN RETURN (SELECT da.term FROM langrefset_active as lr join description_active as da\n" + + " on da.id = lr.referencedcomponentid WHERE lr.acceptabilityid = 900000000000548007 and conceptid = candidate); END//\n" + + "\n" + + "drop function if exists get_cr_FSN//\n" + + "create function get_cr_FSN(candidate bigint) returns varchar(4500)\n" + + "BEGIN RETURN (SELECT term FROM description_active where conceptId = candidate and typeId = 900000000000003001 and languageCode = 'en'); END//\n" + + "\n" + + "drop function if exists get_cr_PercentDefined//\n" + + "create function get_cr_PercentDefined(refset bigint) returns decimal(6, 4)\n" + + "BEGIN SET @refset = refset; SET @refsetSize = (select count(1) from simplerefset_active where refsetId = @refset); " + + "SET @definedCount = (select count(1) from concept_active where definitionStatusId = 900000000000073002 " + + "and id in (select referencedComponentId from simplerefset_active where refsetId = @refset)); " + + "RETURN CONVERT(@definedCount/ @refsetSize * 100,DECIMAL(6,4)); END//\n"; + List result = queryTransformer.transformToStatements(sqlToTest); + assertEquals(6, result.size()); + assertTrue(result.get(0).startsWith("drop function if exists get_cr_ADRS_PT") ); + assertTrue(result.get(5).startsWith("create function get_cr_PercentDefined(refset bigint)")); + } + @Test + public void transformToStatementsUsingDefaultDelimiter() throws BusinessServiceException { + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + String sqlToTest = "SET default_storage_engine=MYISAM/;\n" + + "DROP TABLE IF EXISTS concept_active;\n" + + "CREATE TABLE concept_active ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AS\n" + + "SELECT * FROM concept_ AS concepts WHERE active = 1;\n" + + "create unique index concept_active_id_ix on concept_active(id);\n" + + "create index concept_active_effectivetime_ix on concept_active(effectivetime);\n" + + "create index concept_active_definitionstatusid_ix on concept_active(definitionstatusid);\n" + + "create index concept_active_moduleid_ix on concept_active(moduleid);\n" + + "create index concept_active_active_ix on concept_active(active);"; + List result = queryTransformer.transformToStatements(sqlToTest); + assertEquals(8, result.size()); + assertEquals("create index concept_active_active_ix on concept_active(active)", result.get(7)); + assertEquals("CREATE TABLE concept_active ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AS\n" + + "SELECT * FROM concept_ AS concepts WHERE active = 1", result.get(2)); + } + @Test + public void transformToStatementInvalidDelimiter() { + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + String sqlToTest = "DELIMITER // SET default_storage_engine=MYISAM/;\n" + + "DROP TABLE IF EXISTS concept_active;"; + BusinessServiceException exception = assertThrows(BusinessServiceException.class, () -> { + queryTransformer.transformToStatements(sqlToTest); + }); + + String expectedMessagePattern = "SQL statements not ending with // SET default_storage_engine=MYISAM"; + String actualMessage = exception.getMessage(); + assertTrue(actualMessage.startsWith(expectedMessagePattern)); + + } + + @Test + public void transformSql() throws ConfigurationException { + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + String sql = "insert into res_concepts_edited\n" + + "\tselect distinct id\n" + + "\tfrom .concept_\n" + + "\twhere id in ( \n" + + "\t\tselect id from .concept_\n" + + " union \n" + + " select conceptid from .description_\n" + + " union \n" + + " select conceptid from .textdefinition_\n" + + " union \n" + + " select sourceid from .stated_relationship_\n" + + " union\n" + + " select referencedcomponentid from .owlexpressionrefset_\n" + + " union\n" + + " select b.conceptid from .langrefset_ a \n" + + " left join .description_ b on a.referencedcomponentid=b.id\n" + + " union \n" + + " select referencedcomponentid from .attributevaluerefset_\n" + + " where refsetid = '900000000000489007'\n" + + " union \n" + + " select a.conceptid from .description_ a\n" + + " join .attributevaluerefset_ b on a.id = b.referencedcomponentid\n" + + " where b.refsetid = '900000000000490003'\n" + + "\tunion\n" + + " select referencedcomponentid\n" + + " from .associationrefset_\n" + + " where refsetid in ('900000000000523009','900000000000526001','900000000000527005','900000000000530003','1186924009','1186921001')\n" + + "\tunion\n" + + " select a.conceptid from .description_ a \n" + + " join .associationrefset_ b on a.id = b.referencedcomponentid \n" + + " where b.refsetid = '900000000000531004'\n" + + "\tunion\n" + + " select referencedcomponentid, from .simplerefset_\n" + + "\tunion\n" + + " select referencedcomponentid from .simplemaprefset_ and RLIKE Binary '[[:<:]]hexachlorophene[[:>:]]'\n" + + " )"; + String[] sqlParts = {sql}; + String testAssertionId = "xyz"; + String testQaResult = "abc"; + when(config.getProspectiveVersion()).thenReturn("rvf_au_20221231_230619110027"); + when(config.getPreviousVersion()).thenReturn(String.valueOf(10)); + Map configMap = Map.of("qa_result",testQaResult, + "", String.valueOf(testAssertionId)); + List result = queryTransformer.transformSql(sqlParts,config, configMap); + assertEquals(1,result.size()); + assertTrue(!result.get(0).contains("")); + assertTrue(!result.get(0).contains("")); + assertTrue(result.get(0).contains("rvf_au_20221231_230619110027.description_d")); + assertTrue(result.get(0).contains("'\\bhexachlorophene\\b'")); + assertTrue(result.get(0).contains(testAssertionId)); + } + + + @Test + public void transformSqlMissingConfig() { + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + String sqlToTest = "SET default_storage_engine=MYISAM/;\n" + + "DROP TABLE IF EXISTS concept_active;"; + String[] sqlParts = {sqlToTest}; + ConfigurationException exception = assertThrows(ConfigurationException.class, () -> { + queryTransformer.transformSql(sqlParts,config, Collections.emptyMap()); + }); + + String expectedMessagePattern = "Failed to find rvf db schema for null"; + String actualMessage = exception.getMessage(); + assertTrue(actualMessage.startsWith(expectedMessagePattern)); + + } + +}