Merge pull request #32 from JULIELab/v3.2

V3.2

julielab-neo4j-plugins-concepts-representation/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>de.julielab</groupId>
 		<artifactId>julielab-neo4j-server-plugins</artifactId>
-		<version>3.1.0</version>
+		<version>3.2.0</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<artifactId>julielab-neo4j-plugins-concepts-representation</artifactId>

julielab-neo4j-plugins-concepts/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>de.julielab</groupId>
         <artifactId>julielab-neo4j-server-plugins</artifactId>
-        <version>3.1.0</version>
+        <version>3.2.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>julielab-neo4j-plugins-concepts</artifactId>
@@ -33,7 +33,7 @@
         <dependency>
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>
-            <version>2.7</version>
+            <version>2.11.0</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
@@ -46,7 +46,7 @@
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
-            <version>2.8.2</version>
+            <version>2.8.9</version>
         </dependency>
         <dependency>
             <groupId>org.assertj</groupId>

julielab-neo4j-plugins-concepts/src/main/java/de/julielab/neo4j/plugins/concepts/ConceptAggregateManager.java

@@ -59,6 +59,7 @@ public class ConceptAggregateManager {
     public static final String KEY_SKIP_EXISTING_PROPERTIES = "skip_existing_properties";
     public static final String KEY_ALLOWED_MAPPING_TYPES = "allowedMappingTypes";
     public static final String KEY_COPY_PROPERTIES = "copy_properties";
+    public static final String KEY_NAME_PROPERTY = "name_property";
     public static final String RET_KEY_NUM_AGGREGATES = "numAggregates";
     public static final String RET_KEY_NUM_ELEMENTS = "numElements";
     public static final String RET_KEY_NUM_PROPERTIES = "numProperties";
@@ -192,13 +193,14 @@ static void insertAggregateConcept(Transaction tx, ImportConcept jsonConcept,
      *
      * @param graphDb               The graph database to work on.
      * @param nodeLabels
+     * @param nameProperty
      * @param aggregatedLabels
      * @param copyProperties
      * @return
      */
-    public static int buildAggregatesForEqualNames(GraphDatabaseService graphDb, List<Label> nodeLabels, List<Label> aggregatedLabels, String[] copyProperties, Log log) {
+    public static int buildAggregatesForEqualNames(GraphDatabaseService graphDb, List<Label> nodeLabels, String nameProperty, List<Label> aggregatedLabels, String[] copyProperties, Log log) {
         int createdAggregates = 0;
-        Comparator<Node> nodeNameComparator = Comparator.comparing(n -> ((String) n.getProperty(PROP_PREF_NAME)).toLowerCase().replaceAll("\\s+", ""));
+        Comparator<Node> nodeNameComparator = Comparator.comparing(n -> ((String) n.getProperty(nameProperty)));
 
         // Sort the nodes with the target according to their preferred name
         log.info("Acquiring all non-aggregate nodes with labels %s", nodeLabels);
@@ -607,13 +609,10 @@ public static void copyAggregateProperties(Node aggregate, boolean skipExistingP
                     Object property = term.getProperty(copyProperty);
                     if (property.getClass().isArray()) {
                         final Object[] mergedValue = mergeArrayValue(newAggregateProperties.getOrDefault(copyProperty, null), JulieNeo4jUtilities.convertArray(property));
-                        // TODO
-//                        aggregate.setProperty(copyProperty, mergedValue);
                         newAggregateProperties.put(copyProperty, mergedValue);
                     } else {
                         if (!newAggregateProperties.containsKey(copyProperty))
                             newAggregateProperties.put(copyProperty, property);
-//                        setNonNullNodeProperty(aggregate, copyProperty, property);
                         Object aggregateProperty = newAggregateProperties.get(copyProperty);
                         if (!aggregateProperty.equals(property)) {
                             divergentProperties.add(copyProperty);
@@ -748,39 +747,20 @@ public Response buildAggregatesByPreferredName(String jsonParameterObject, @Cont
             if (!parameterMap.containsKey(KEY_LABELS))
                 throw new IllegalArgumentException("Parameter '" + KEY_LABELS + "' not specified.");
             List<Label> aggregatedLabels = List.of(AGGREGATE_EQUAL_NAMES);
+            String nameProperty = PROP_PREF_NAME;
+            if (parameterMap.containsKey(KEY_NAME_PROPERTY))
+                nameProperty = (String) parameterMap.get(KEY_NAME_PROPERTY);
             if (parameterMap.containsKey(KEY_AGGREGATED_LABELS))
                 aggregatedLabels = ((List<String>) parameterMap.get(KEY_AGGREGATED_LABELS)).stream().map(Label::label).collect(Collectors.toList());
             List<String> copyProperties = List.of(PROP_PREF_NAME, PROP_SYNONYMS);
             if (parameterMap.containsKey(KEY_COPY_PROPERTIES))
                 copyProperties = ((List<String>) parameterMap.get(KEY_COPY_PROPERTIES));
             List<Label> targetLabels = ((List<String>) parameterMap.get(KEY_LABELS)).stream().map(Label::label).collect(Collectors.toList());
-            log.info("Creating equal-name-aggregates for concepts with label %s and assigning them label", targetLabels, aggregatedLabels);
+            log.info("Creating equal-name-aggregates regarding the '%s' property for concepts with label %s and assigning them label %s", nameProperty, targetLabels, aggregatedLabels);
             GraphDatabaseService graphDb = dbms.database(DEFAULT_DATABASE_NAME);
             int createdAggregates;
             log.info("Beginning transaction for the creation of equal-name aggregates.");
-            createdAggregates = ConceptAggregateManager.buildAggregatesForEqualNames(graphDb, targetLabels, aggregatedLabels, copyProperties.toArray(String[]::new), log);
-//            log.info("Finding nodes");
-//            List<ConceptCoordinates> uniquelyNamedNodes = new ArrayList<>();
-//            try (Transaction tx = graphDb.beginTx()) {
-//                ResourceIterable<Node> nodeIterable = () -> tx.findNodes(targetLabel);
-//                for (Node n : nodeIterable) {
-//                    // omit aggregate nodes
-//                    if (n.hasLabel(AGGREGATE))
-//                        continue;
-//                    // omit nodes that are element of an equal-name aggregate
-//                    if (StreamSupport.stream(n.getRelationships(Direction.INCOMING, ConceptEdgeTypes.HAS_ELEMENT).spliterator(), false).anyMatch(r -> r.getOtherNode(n).hasLabel(AGGREGATE_EQUAL_NAMES)))
-//                        continue;
-//                    uniquelyNamedNodes.add(new ConceptCoordinates((String) n.getProperty(PROP_ORG_ID), (String) n.getProperty(PROP_ORG_SRC), CoordinateType.OSRC));
-//                }
-//
-//            }
-//            log.info("Adding label %s for all nodes that have a unique name and are not part of an equal-name aggregate.", AGGREGATE_EQUAL_NAMES);
-//            try (Transaction tx = graphDb.beginTx()) {
-//                ConceptAggregateManager.addUniqueNameLabels(tx, targetLabels, log);
-//                log.info("Finished labeling unique-named concepts with label %s.", targetLabels);
-//                log.info("Committing transaction for the creation of unique-named concepts.");
-//                tx.commit();
-//            }
+            createdAggregates = ConceptAggregateManager.buildAggregatesForEqualNames(graphDb, targetLabels, nameProperty, aggregatedLabels, copyProperties.toArray(String[]::new), log);
             log.info("Process for the creation of equal-name aggregates has finished.");
             return Response.ok(createdAggregates).build();
         } catch (IOException e) {

julielab-neo4j-plugins-concepts/src/test/java/de/julielab/neo4j/plugins/ExportIntegrationTest.java

@@ -24,7 +24,8 @@
 import static org.neo4j.logging.FormattedLogFormat.PLAIN;
 
 public class ExportIntegrationTest {
-private static Log log;
+    private static Log log;
+
     static {
         Log4jLogProvider log4jLogProvider = new Log4jLogProvider(LogConfig.createBuilder(System.out, Level.INFO)
                 .withFormat(PLAIN)
@@ -63,8 +64,8 @@ public void exportIdMapping() throws IOException {
                     "CONCEPT1\ttid1\n",
                     "CONCEPT4\ttid4\n",
                     "CONCEPT3\ttid3\n");
-for (var item : expectedItems)
-    assertThat(mapping).contains(item);
+            for (var item : expectedItems)
+                assertThat(mapping).contains(item);
         }
     }
 }

julielab-neo4j-plugins-concepts/src/test/java/de/julielab/neo4j/plugins/ExportTest.java

@@ -7,6 +7,7 @@
 import de.julielab.neo4j.plugins.datarepresentation.ConceptCoordinates;
 import de.julielab.neo4j.plugins.datarepresentation.ImportConcept;
 import de.julielab.neo4j.plugins.datarepresentation.ImportConcepts;
+import de.julielab.neo4j.plugins.datarepresentation.constants.ConceptConstants;
 import de.julielab.neo4j.plugins.datarepresentation.util.ConceptsJsonSerializer;
 import de.julielab.neo4j.plugins.test.TestUtilities;
 import org.apache.commons.io.IOUtils;
@@ -25,10 +26,13 @@
 import java.io.ByteArrayOutputStream;
 import java.io.OutputStream;
 import java.lang.reflect.Method;
+import java.util.Base64;
+import java.util.zip.GZIPInputStream;
 
 import static de.julielab.neo4j.plugins.concepts.ConceptLabel.CONCEPT;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.neo4j.configuration.GraphDatabaseSettings.DEFAULT_DATABASE_NAME;
 
 public class ExportTest {
@@ -192,4 +196,91 @@ public void createIdMappingOneFacetOriginalId() throws Exception {
         assertEquals(3, countMatches);
     }
 
+    @Test
+    public void exportLingpipeDictionary() throws Exception {
+        new Indexes(graphDBMS).createIndexes((String) null);
+        ImportConcepts importConcepts = ConceptManagerTest.getTestConcepts(10);
+        ConceptManager tm = new ConceptManager(graphDBMS, LogUtilities.getLogger(ConceptManager.class));
+        tm.insertConcepts(new ByteArrayInputStream(ConceptsJsonSerializer.toJson(importConcepts).getBytes(UTF_8)));
+
+        Export export = new Export(graphDBMS);
+
+        // create mapping file contents for original Ids
+        final String gzippedBase64EncodedDictionary = export.exportLingpipeDictionary(CONCEPT.name(), null, ConceptConstants.PROP_SRC_IDS + 0, false, true, LogUtilities.getLogger(Export.class));
+        final byte[] gzippedBytes = Base64.getDecoder().decode(gzippedBase64EncodedDictionary);
+        String dictionary;
+        try (final GZIPInputStream is = new GZIPInputStream(new ByteArrayInputStream(gzippedBytes))) {
+            dictionary = IOUtils.toString(is, UTF_8);
+        }
+        assertTrue(dictionary.contains("prefname0\tCONCEPT0"));
+        assertTrue(dictionary.contains("prefname1\tCONCEPT1"));
+        assertTrue(dictionary.contains("prefname2\tCONCEPT2"));
+        assertTrue(dictionary.contains("prefname3\tCONCEPT3"));
+        assertTrue(dictionary.contains("prefname4\tCONCEPT4"));
+        assertTrue(dictionary.contains("prefname5\tCONCEPT5"));
+        assertTrue(dictionary.contains("prefname6\tCONCEPT6"));
+        assertTrue(dictionary.contains("prefname7\tCONCEPT7"));
+        assertTrue(dictionary.contains("prefname8\tCONCEPT8"));
+        assertTrue(dictionary.contains("prefname9\tCONCEPT9"));
+    }
+
+    @Test
+    public void exportLingpipeDictionary2() throws Exception {
+        new Indexes(graphDBMS).createIndexes((String) null);
+        ImportConcepts importConcepts = ConceptManagerTest.getTestConcepts(10);
+        ConceptManager tm = new ConceptManager(graphDBMS, LogUtilities.getLogger(ConceptManager.class));
+        tm.insertConcepts(new ByteArrayInputStream(ConceptsJsonSerializer.toJson(importConcepts).getBytes(UTF_8)));
+
+        Export export = new Export(graphDBMS);
+
+        // create mapping file contents for original Ids
+        final String gzippedBase64EncodedDictionary = export.exportLingpipeDictionary(CONCEPT.name(), null, ConceptConstants.PROP_SRC_IDS + 0, true, true, LogUtilities.getLogger(Export.class));
+        final byte[] gzippedBytes = Base64.getDecoder().decode(gzippedBase64EncodedDictionary);
+        String dictionary;
+        try (final GZIPInputStream is = new GZIPInputStream(new ByteArrayInputStream(gzippedBytes))) {
+            dictionary = IOUtils.toString(is, UTF_8);
+        }
+        assertTrue(dictionary.contains("prefname0\tTEST_DATA:CONCEPT0"));
+        assertTrue(dictionary.contains("prefname1\tTEST_DATA:CONCEPT1"));
+        assertTrue(dictionary.contains("prefname2\tTEST_DATA:CONCEPT2"));
+        assertTrue(dictionary.contains("prefname3\tTEST_DATA:CONCEPT3"));
+        assertTrue(dictionary.contains("prefname4\tTEST_DATA:CONCEPT4"));
+        assertTrue(dictionary.contains("prefname5\tTEST_DATA:CONCEPT5"));
+        assertTrue(dictionary.contains("prefname6\tTEST_DATA:CONCEPT6"));
+        assertTrue(dictionary.contains("prefname7\tTEST_DATA:CONCEPT7"));
+        assertTrue(dictionary.contains("prefname8\tTEST_DATA:CONCEPT8"));
+        assertTrue(dictionary.contains("prefname9\tTEST_DATA:CONCEPT9"));
+    }
+
+    @Test
+    public void exportLingpipeDictionary3() throws Exception {
+        new Indexes(graphDBMS).createIndexes((String) null);
+        ImportConcepts importConcepts = ConceptManagerTest.getTestConcepts(10);
+        for (var c : importConcepts.getConceptsAsList())
+            c.coordinates.originalSource = "ORG_SRC";
+        ConceptManager tm = new ConceptManager(graphDBMS, LogUtilities.getLogger(ConceptManager.class));
+        tm.insertConcepts(new ByteArrayInputStream(ConceptsJsonSerializer.toJson(importConcepts).getBytes(UTF_8)));
+
+        Export export = new Export(graphDBMS);
+
+        // create mapping file contents for original Ids
+        final String gzippedBase64EncodedDictionary = export.exportLingpipeDictionary(CONCEPT.name(), null, "[\""+ConceptConstants.PROP_SRC_IDS + 0 + "\",\""+ConceptConstants.PROP_ORG_ID + "\"]", true, true, LogUtilities.getLogger(Export.class));
+        final byte[] gzippedBytes = Base64.getDecoder().decode(gzippedBase64EncodedDictionary);
+        String dictionary;
+        try (final GZIPInputStream is = new GZIPInputStream(new ByteArrayInputStream(gzippedBytes))) {
+            dictionary = IOUtils.toString(is, UTF_8);
+        }
+        assertTrue(dictionary.contains("prefname5	TEST_DATA:CONCEPT5||ORG_SRC:CONCEPT5"));
+        assertTrue(dictionary.contains("prefname6	TEST_DATA:CONCEPT6||ORG_SRC:CONCEPT6"));
+        assertTrue(dictionary.contains("prefname3	TEST_DATA:CONCEPT3||ORG_SRC:CONCEPT3"));
+        assertTrue(dictionary.contains("prefname4	TEST_DATA:CONCEPT4||ORG_SRC:CONCEPT4"));
+        assertTrue(dictionary.contains("prefname9	TEST_DATA:CONCEPT9||ORG_SRC:CONCEPT9"));
+        assertTrue(dictionary.contains("prefname1	TEST_DATA:CONCEPT1||ORG_SRC:CONCEPT1"));
+        assertTrue(dictionary.contains("prefname2	TEST_DATA:CONCEPT2||ORG_SRC:CONCEPT2"));
+        assertTrue(dictionary.contains("prefname7	TEST_DATA:CONCEPT7||ORG_SRC:CONCEPT7"));
+        assertTrue(dictionary.contains("prefname8	TEST_DATA:CONCEPT8||ORG_SRC:CONCEPT8"));
+        assertTrue(dictionary.contains("prefname0	TEST_DATA:CONCEPT0||ORG_SRC:CONCEPT0"));
+
+    }
+
 }

julielab-neo4j-plugins-utilities/pom.xml

@@ -3,7 +3,7 @@
   <parent>
     <groupId>de.julielab</groupId>
     <artifactId>julielab-neo4j-server-plugins</artifactId>
-    <version>3.1.0</version>
+    <version>3.2.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
   <artifactId>julielab-neo4j-plugins-utilities</artifactId>

pom.xml

@@ -7,7 +7,7 @@
 		<version>2.5.0</version>
 	</parent>
 	<artifactId>julielab-neo4j-server-plugins</artifactId>
-	<version>3.1.0</version>
+	<version>3.2.0</version>
 	<name>JULIE Lab Neo4j Server Plugins</name>
 	<build>
 		<pluginManagement>