diff --git a/pom.xml b/pom.xml index 90bc562..369f28c 100644 --- a/pom.xml +++ b/pom.xml @@ -51,10 +51,10 @@ 2.6.0 - 2.2.5-SNAPSHOT + 2.2.8 0.5.4 0.98.8-hadoop2 - 2.0.1 + 3.0.5 1.0.0 2.1 2.18.1 @@ -148,22 +148,47 @@ jna 4.0.0 - + + + + org.neo4j + neo4j + ${neo4j.version} + + + com.orientechnologies orientdb-graphdb diff --git a/src/main/java/eu/socialsensor/graphdatabases/GraphDatabaseBase.java b/src/main/java/eu/socialsensor/graphdatabases/GraphDatabaseBase.java index d4992e0..28732b9 100644 --- a/src/main/java/eu/socialsensor/graphdatabases/GraphDatabaseBase.java +++ b/src/main/java/eu/socialsensor/graphdatabases/GraphDatabaseBase.java @@ -4,7 +4,6 @@ import java.util.Set; import org.neo4j.graphdb.Transaction; -import org.neo4j.kernel.GraphDatabaseAPI; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; @@ -98,7 +97,7 @@ public void findAllNodeNeighbours() { } } finally {//TODO fix this if(GraphDatabaseType.NEO4J == type) { - ((Transaction) tx).finish(); + ((Transaction) tx).terminate(); } } } @@ -107,7 +106,7 @@ public void findAllNodeNeighbours() { public void findNodesOfAllEdges() { Object tx = null; if(GraphDatabaseType.NEO4J == type) {//TODO fix this - tx = ((GraphDatabaseAPI) ((Neo4jGraphDatabase) this).neo4jGraph).tx().unforced().begin(); + tx = ((Neo4jGraphDatabase)this).neo4jGraph.beginTx(); } try { @@ -163,7 +162,7 @@ public void shortestPaths(Set nodes) { } } finally {//TODO fix this if(GraphDatabaseType.NEO4J == type) { - ((Transaction) tx).finish(); + ((Transaction) tx).terminate(); } } } diff --git a/src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java b/src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java index d3e86d6..784b73a 100644 --- a/src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java +++ b/src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java @@ -1,7 +1,5 @@ package eu.socialsensor.graphdatabases; -import com.google.common.collect.Iterables; - import eu.socialsensor.insert.Insertion; import eu.socialsensor.insert.Neo4jMassiveInsertion; import eu.socialsensor.insert.Neo4jSingleInsertion; @@ -11,27 +9,15 @@ import org.neo4j.graphalgo.GraphAlgoFactory; import org.neo4j.graphalgo.PathFinder; -import org.neo4j.graphdb.Direction; -import org.neo4j.graphdb.DynamicLabel; -import org.neo4j.graphdb.GraphDatabaseService; -import org.neo4j.graphdb.Label; -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Path; -import org.neo4j.graphdb.Relationship; -import org.neo4j.graphdb.RelationshipType; -import org.neo4j.graphdb.ResourceIterable; -import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.*; import org.neo4j.graphdb.factory.GraphDatabaseFactory; import org.neo4j.graphdb.schema.Schema; -import org.neo4j.helpers.collection.IteratorUtil; -import org.neo4j.kernel.GraphDatabaseAPI; -import org.neo4j.kernel.TransactionBuilder; -import org.neo4j.kernel.Traversal; -import org.neo4j.tooling.GlobalGraphOperations; +import org.neo4j.helpers.collection.Iterators; import org.neo4j.unsafe.batchinsert.BatchInserter; import org.neo4j.unsafe.batchinsert.BatchInserters; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -50,6 +36,7 @@ public class Neo4jGraphDatabase extends GraphDatabaseBase, Iterator, Node, Relationship> { protected GraphDatabaseService neo4jGraph = null; + private Schema schema = null; private BatchInserter inserter = null; @@ -69,8 +56,8 @@ public Neo4jGraphDatabase(File dbStorageDirectoryIn) @Override public void open() { - neo4jGraph = new GraphDatabaseFactory().newEmbeddedDatabase(dbStorageDirectory.getAbsolutePath()); - try (final Transaction tx = beginUnforcedTransaction()) + neo4jGraph = new GraphDatabaseFactory().newEmbeddedDatabase(dbStorageDirectory); + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -88,8 +75,9 @@ public void open() @Override public void createGraphForSingleLoad() { - neo4jGraph = new GraphDatabaseFactory().newEmbeddedDatabase(dbStorageDirectory.getAbsolutePath()); - try (final Transaction tx = beginUnforcedTransaction()) + neo4jGraph = new GraphDatabaseFactory().newEmbeddedDatabase(dbStorageDirectory); + + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -118,7 +106,11 @@ public void createGraphForMassiveLoad() config.put("neostore.propertystore.db.mapped_memory", "250M"); config.put("neostore.propertystore.db.strings.mapped_memory", "250M"); - inserter = BatchInserters.inserter(dbStorageDirectory.getAbsolutePath(), config); + try { + inserter = BatchInserters.inserter(dbStorageDirectory, config); + } catch (IOException e) { + e.printStackTrace(); + } createDeferredSchema(); } @@ -188,28 +180,22 @@ public void shutdownMassiveGraph() @Override public void shortestPath(Node n1, Integer i) { + PathExpander pathExpander = PathExpanderBuilder.allTypesAndDirections().add(Neo4jGraphDatabase.RelTypes.SIMILAR,Direction.BOTH).build(); PathFinder finder - = GraphAlgoFactory.shortestPath(Traversal.expanderForTypes(Neo4jGraphDatabase.RelTypes.SIMILAR), 5); + = GraphAlgoFactory.shortestPath(pathExpander, 5); Node n2 = getVertex(i); Path path = finder.findSinglePath(n1, n2); - - } - - //TODO can unforced option be pulled into configuration? - private Transaction beginUnforcedTransaction() { - final TransactionBuilder builder = ((GraphDatabaseAPI) neo4jGraph).tx().unforced(); - return builder.begin(); } @Override public int getNodeCount() { int nodeCount = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - nodeCount = IteratorUtil.count(GlobalGraphOperations.at(neo4jGraph).getAllNodes()); + nodeCount = (int)Iterators.count(neo4jGraph.getAllNodes().iterator()); tx.success(); } catch (Exception e) @@ -226,12 +212,11 @@ public int getNodeCount() public Set getNeighborsIds(int nodeId) { Set neighbors = new HashSet(); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - Node n = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_ID, String.valueOf(nodeId)).iterator() - .next(); + Node n = neo4jGraph.findNodes(NODE_LABEL, NODE_ID, String.valueOf(nodeId)).next(); for (Relationship relationship : n.getRelationships(RelTypes.SIMILAR, Direction.OUTGOING)) { Node neighbour = relationship.getOtherNode(n); @@ -254,12 +239,11 @@ public Set getNeighborsIds(int nodeId) public double getNodeWeight(int nodeId) { double weight = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - Node n = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_ID, String.valueOf(nodeId)).iterator() - .next(); + Node n = neo4jGraph.findNodes(NODE_LABEL, NODE_ID, String.valueOf(nodeId)).next(); weight = getNodeOutDegree(n); tx.success(); } @@ -276,13 +260,13 @@ public double getNodeWeight(int nodeId) public double getNodeInDegree(Node node) { Iterable rel = node.getRelationships(Direction.OUTGOING, RelTypes.SIMILAR); - return (double) (IteratorUtil.count(rel)); + return (double) (Iterators.count(rel.iterator())); } public double getNodeOutDegree(Node node) { Iterable rel = node.getRelationships(Direction.INCOMING, RelTypes.SIMILAR); - return (double) (IteratorUtil.count(rel)); + return (double) (Iterators.count(rel.iterator())); } @Override @@ -291,11 +275,11 @@ public void initCommunityProperty() int communityCounter = 0; // maybe commit changes every 1000 transactions? - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - for (Node n : GlobalGraphOperations.at(neo4jGraph).getAllNodes()) + for (Node n : neo4jGraph.getAllNodes()) { n.setProperty(NODE_COMMUNITY, communityCounter); n.setProperty(COMMUNITY, communityCounter); @@ -315,14 +299,15 @@ public void initCommunityProperty() public Set getCommunitiesConnectedToNodeCommunities(int nodeCommunities) { Set communities = new HashSet(); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - ResourceIterable nodes = neo4jGraph.findNodesByLabelAndProperty(Neo4jGraphDatabase.NODE_LABEL, + ResourceIterator nodes = neo4jGraph.findNodes(Neo4jGraphDatabase.NODE_LABEL, NODE_COMMUNITY, nodeCommunities); - for (Node n : nodes) + while(nodes.hasNext()) { + Node n = nodes.next(); for (Relationship r : n.getRelationships(RelTypes.SIMILAR, Direction.OUTGOING)) { Node neighbour = r.getOtherNode(n); @@ -346,13 +331,14 @@ public Set getCommunitiesConnectedToNodeCommunities(int nodeCommunities public Set getNodesFromCommunity(int community) { Set nodes = new HashSet(); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - ResourceIterable iter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, COMMUNITY, community); - for (Node n : iter) + ResourceIterator iter = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, community); + while(iter.hasNext()) { + Node n = iter.next(); String nodeIdString = (String) (n.getProperty(NODE_ID)); nodes.add(Integer.valueOf(nodeIdString)); } @@ -372,14 +358,15 @@ public Set getNodesFromNodeCommunity(int nodeCommunity) { Set nodes = new HashSet(); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - ResourceIterable iter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_COMMUNITY, + ResourceIterator iter = neo4jGraph.findNodes(NODE_LABEL, NODE_COMMUNITY, nodeCommunity); - for (Node n : iter) + while(iter.hasNext()) { + Node n = iter.next(); String nodeIdString = (String) (n.getProperty(NODE_ID)); nodes.add(Integer.valueOf(nodeIdString)); } @@ -399,23 +386,26 @@ public Set getNodesFromNodeCommunity(int nodeCommunity) public double getEdgesInsideCommunity(int nodeCommunity, int communityNodes) { double edges = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - ResourceIterable nodes = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_COMMUNITY, + ResourceIterator nodes = neo4jGraph.findNodes(NODE_LABEL, NODE_COMMUNITY, nodeCommunity); - ResourceIterable comNodes = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, COMMUNITY, + ResourceIterator comNodes = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, communityNodes); - for (Node node : nodes) + while(nodes.hasNext()) { + Node node = nodes.next(); Iterable relationships = node.getRelationships(RelTypes.SIMILAR, Direction.OUTGOING); for (Relationship r : relationships) { Node neighbor = r.getOtherNode(node); - if (Iterables.contains(comNodes, neighbor)) - { - edges++; + while(comNodes.hasNext()) { + Node comNode = comNodes.next(); + if(comNode.equals(neighbor)){ + edges++; + } } } } @@ -435,17 +425,13 @@ public double getEdgesInsideCommunity(int nodeCommunity, int communityNodes) public double getCommunityWeight(int community) { double communityWeight = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - ResourceIterable iter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, COMMUNITY, community); - if (Iterables.size(iter) > 1) - { - for (Node n : iter) - { - communityWeight += getNodeOutDegree(n); - } + ResourceIterator iter = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, community); + while(iter.hasNext()){ + communityWeight += getNodeOutDegree(iter.next()); } tx.success(); } @@ -463,18 +449,14 @@ public double getCommunityWeight(int community) public double getNodeCommunityWeight(int nodeCommunity) { double nodeCommunityWeight = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - ResourceIterable iter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_COMMUNITY, + ResourceIterator iter = neo4jGraph.findNodes(NODE_LABEL, NODE_COMMUNITY, nodeCommunity); - if (Iterables.size(iter) > 1) - { - for (Node n : iter) - { - nodeCommunityWeight += getNodeOutDegree(n); - } + while(iter.hasNext()){ + nodeCommunityWeight += getNodeOutDegree(iter.next()); } tx.success(); } @@ -491,15 +473,14 @@ public double getNodeCommunityWeight(int nodeCommunity) @Override public void moveNode(int nodeCommunity, int toCommunity) { - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - ResourceIterable fromIter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_COMMUNITY, + ResourceIterator iter = neo4jGraph.findNodes(NODE_LABEL, NODE_COMMUNITY, nodeCommunity); - for (Node node : fromIter) - { - node.setProperty(COMMUNITY, toCommunity); + while(iter.hasNext()){ + iter.next().setProperty(COMMUNITY, toCommunity); } tx.success(); } @@ -516,11 +497,11 @@ public double getGraphWeightSum() { int edgeCount = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - edgeCount = IteratorUtil.count(GlobalGraphOperations.at(neo4jGraph).getAllRelationships()); + edgeCount = (int)Iterators.count(neo4jGraph.getAllRelationships().iterator()); tx.success(); } catch (Exception e) @@ -539,11 +520,11 @@ public int reInitializeCommunities() Map initCommunities = new HashMap(); int communityCounter = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - for (Node n : GlobalGraphOperations.at(neo4jGraph).getAllNodes()) + for (Node n : neo4jGraph.getAllNodes()) { Integer communityId = (Integer) (n.getProperty(COMMUNITY)); if (!initCommunities.containsKey(communityId)) @@ -572,12 +553,11 @@ public int getCommunity(int nodeCommunity) { Integer community = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - Node node = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_COMMUNITY, nodeCommunity).iterator() - .next(); + Node node = neo4jGraph.findNode(NODE_LABEL, NODE_COMMUNITY, nodeCommunity); community = (Integer) (node.getProperty(COMMUNITY)); tx.success(); } @@ -595,12 +575,11 @@ public int getCommunity(int nodeCommunity) public int getCommunityFromNode(int nodeId) { Integer community = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - Node node = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_ID, String.valueOf(nodeId)).iterator() - .next(); + Node node = neo4jGraph.findNode(NODE_LABEL, NODE_ID, String.valueOf(nodeId)); community = (Integer) (node.getProperty(COMMUNITY)); tx.success(); } @@ -619,14 +598,13 @@ public int getCommunitySize(int community) { Set nodeCommunities = new HashSet(); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - ResourceIterable nodes = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, COMMUNITY, community); - for (Node n : nodes) - { - Integer nodeCommunity = (Integer) (n.getProperty(COMMUNITY)); + ResourceIterator nodes = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, community); + while(nodes.hasNext()){ + Integer nodeCommunity = (Integer) (nodes.next().getProperty(COMMUNITY)); nodeCommunities.add(nodeCommunity); } tx.success(); @@ -646,17 +624,16 @@ public Map> mapCommunities(int numberOfCommunities) { Map> communities = new HashMap>(); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { for (int i = 0; i < numberOfCommunities; i++) { - ResourceIterable nodesIter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, COMMUNITY, i); + ResourceIterator nodesIter = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, i); List nodes = new ArrayList(); - for (Node n : nodesIter) - { - String nodeIdString = (String) (n.getProperty(NODE_ID)); + while(nodesIter.hasNext()){ + String nodeIdString = (String) (nodesIter.next().getProperty(NODE_ID)); nodes.add(Integer.valueOf(nodeIdString)); } communities.put(i, nodes); @@ -669,19 +646,18 @@ public Map> mapCommunities(int numberOfCommunities) throw new BenchmarkingException("unable to map communities", e); } } - return communities; } @Override public boolean nodeExists(int nodeId) { - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - ResourceIterable nodesIter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_ID, nodeId); - if (nodesIter.iterator().hasNext()) + ResourceIterator nodesIter = neo4jGraph.findNodes(NODE_LABEL, NODE_ID, nodeId); + if (nodesIter.hasNext()) { tx.success(); return true; @@ -700,7 +676,7 @@ public boolean nodeExists(int nodeId) @Override public Iterator getVertexIterator() { - return GlobalGraphOperations.at(neo4jGraph).getAllNodes().iterator(); + return neo4jGraph.getAllNodes().iterator(); } @Override @@ -724,7 +700,7 @@ public Node getOtherVertexFromEdge(Relationship r, Node n) @Override public Iterator getAllEdges() { - return GlobalGraphOperations.at(neo4jGraph).getAllRelationships().iterator(); + return neo4jGraph.getAllRelationships().iterator(); } @Override @@ -773,8 +749,7 @@ public Node nextVertex(Iterator it) public Node getVertex(Integer i) { // note, this probably should be run in the context of an active transaction. - return neo4jGraph.findNodesByLabelAndProperty(Neo4jGraphDatabase.NODE_LABEL, NODE_ID, i.toString()).iterator() - .next(); + return neo4jGraph.findNode(Neo4jGraphDatabase.NODE_LABEL, NODE_ID, i.toString()); } } diff --git a/src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java b/src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java index c44c280..5dea988 100644 --- a/src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java +++ b/src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java @@ -26,6 +26,7 @@ import eu.socialsensor.main.BenchmarkConfiguration; import eu.socialsensor.main.GraphDatabaseType; import eu.socialsensor.utils.Utils; +import org.apache.commons.collections.map.HashedMap; import java.io.File; import java.util.ArrayList; @@ -53,7 +54,7 @@ public OrientGraphDatabase(BenchmarkConfiguration config, File dbStorageDirector { super(GraphDatabaseType.ORIENT_DB, dbStorageDirectoryIn); OGlobalConfiguration.STORAGE_COMPRESSION_METHOD.setValue("nothing"); - this.useLightWeightEdges = config.orientLightweightEdges() == null ? true : config.orientLightweightEdges() + this.useLightWeightEdges = config.orientLightweightEdges() == null ? false : config.orientLightweightEdges() .booleanValue(); } @@ -67,7 +68,7 @@ public void open() @Override public void createGraphForSingleLoad() { - OGlobalConfiguration.STORAGE_KEEP_OPEN.setValue(false); +// OGlobalConfiguration.STORAGE_KEEP_OPEN.setValue(false); graph = getGraph(dbStorageDirectory); createSchema(); } @@ -102,6 +103,7 @@ public void shutdown() { return; } + graph.commit(); graph.shutdown(); graph = null; } @@ -125,9 +127,10 @@ public void shutdownMassiveGraph() public void shortestPath(final Vertex v1, Integer i) { final OrientVertex v2 = (OrientVertex) getVertex(i); - + Map attrs = new HashMap(); + attrs.put("maxDepth",5); List result = new OSQLFunctionShortestPath().execute(graph, - null, null, new Object[] { ((OrientVertex) v1).getRecord(), v2.getRecord(), Direction.OUT, 5 }, + null, null, new Object[] { ((OrientVertex) v1).getRecord(), v2.getRecord(), Direction.OUT, "similar", attrs}, new OBasicCommandContext()); result.size(); @@ -384,11 +387,14 @@ public Object call(final OrientBaseGraph g) g.createKeyIndex(NODE_ID, Vertex.class, new Parameter("type", "UNIQUE_HASH_INDEX"), new Parameter( "keytype", "INTEGER")); - v.createEdgeProperty(Direction.OUT, SIMILAR, OType.LINKBAG); - v.createEdgeProperty(Direction.IN, SIMILAR, OType.LINKBAG); OrientEdgeType similar = g.createEdgeType(SIMILAR); similar.createProperty("out", OType.LINK, v); similar.createProperty("in", OType.LINK, v); + + v.createEdgeProperty(Direction.OUT, SIMILAR, OType.LINKBAG); + v.createEdgeProperty(Direction.IN, SIMILAR, OType.LINKBAG); + + g.createKeyIndex(COMMUNITY, Vertex.class, new Parameter("type", "NOTUNIQUE_HASH_INDEX"), new Parameter("keytype", "INTEGER")); g.createKeyIndex(NODE_COMMUNITY, Vertex.class, new Parameter("type", "NOTUNIQUE_HASH_INDEX"), diff --git a/src/main/java/eu/socialsensor/insert/Neo4jSingleInsertion.java b/src/main/java/eu/socialsensor/insert/Neo4jSingleInsertion.java index 7ba92f7..62bef0f 100644 --- a/src/main/java/eu/socialsensor/insert/Neo4jSingleInsertion.java +++ b/src/main/java/eu/socialsensor/insert/Neo4jSingleInsertion.java @@ -4,12 +4,10 @@ import java.util.HashMap; import java.util.Map; -import org.neo4j.cypher.javacompat.ExecutionEngine; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.ResourceIterator; import org.neo4j.graphdb.Transaction; -import org.neo4j.kernel.GraphDatabaseAPI; import eu.socialsensor.graphdatabases.Neo4jGraphDatabase; import eu.socialsensor.main.BenchmarkingException; @@ -26,27 +24,27 @@ public class Neo4jSingleInsertion extends InsertionBase { private final GraphDatabaseService neo4jGraph; - private final ExecutionEngine engine; +// private final ExecutionEngine engine; public Neo4jSingleInsertion(GraphDatabaseService neo4jGraph, File resultsPath) { super(GraphDatabaseType.NEO4J, resultsPath); this.neo4jGraph = neo4jGraph; - engine = new ExecutionEngine(this.neo4jGraph); +// engine = new ExecutionEngine(this.neo4jGraph,null); } public Node getOrCreate(String nodeId) { Node result = null; - try(final Transaction tx = ((GraphDatabaseAPI) neo4jGraph).tx().unforced().begin()) + try(final Transaction tx = (neo4jGraph.beginTx())) { try { String queryString = "MERGE (n:Node {nodeId: {nodeId}}) RETURN n"; Map parameters = new HashMap(); parameters.put("nodeId", nodeId); - ResourceIterator resultIterator = engine.execute(queryString, parameters).columnAs("n"); + ResourceIterator resultIterator = neo4jGraph.execute(queryString, parameters).columnAs("n"); result = resultIterator.next(); tx.success(); } @@ -63,7 +61,7 @@ public Node getOrCreate(String nodeId) @Override public void relateNodes(Node src, Node dest) { - try (final Transaction tx = ((GraphDatabaseAPI) neo4jGraph).tx().unforced().begin()) + try (final Transaction tx = (neo4jGraph.beginTx())) { try { diff --git a/src/main/java/eu/socialsensor/insert/OrientMassiveInsertion.java b/src/main/java/eu/socialsensor/insert/OrientMassiveInsertion.java index 6d2a1e7..357b0a0 100644 --- a/src/main/java/eu/socialsensor/insert/OrientMassiveInsertion.java +++ b/src/main/java/eu/socialsensor/insert/OrientMassiveInsertion.java @@ -1,7 +1,11 @@ package eu.socialsensor.insert; import com.orientechnologies.orient.core.config.OGlobalConfiguration; +import com.orientechnologies.orient.core.db.record.OIdentifiable; +import com.orientechnologies.orient.core.index.OIndex; import com.orientechnologies.orient.graph.batch.OGraphBatchInsertBasic; +import com.tinkerpop.blueprints.TransactionalGraph; +import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import eu.socialsensor.main.GraphDatabaseType; @@ -13,13 +17,16 @@ * @author Alexander Patrikalakis * */ -public class OrientMassiveInsertion extends InsertionBase implements Insertion +public class OrientMassiveInsertion extends InsertionBase implements Insertion { private static final int ESTIMATED_ENTRIES = 1000000; private static final int AVERAGE_NUMBER_OF_EDGES_PER_NODE = 40; private static final int NUMBER_OF_ORIENT_CLUSTERS = 16; private final OGraphBatchInsertBasic graph; + protected final OrientGraphNoTx orientGraph; + protected final OIndex index; + public OrientMassiveInsertion(final String url) { super(GraphDatabaseType.ORIENT_DB, null /* resultsPath */); @@ -32,29 +39,80 @@ public OrientMassiveInsertion(final String url) } transactionlessGraph.shutdown(); + orientGraph = new OrientGraphNoTx(url); + this.index = this.orientGraph.getRawGraph().getMetadata().getIndexManager().getIndex("V.nodeId"); + graph = new OGraphBatchInsertBasic(url); graph.setAverageEdgeNumberPerNode(AVERAGE_NUMBER_OF_EDGES_PER_NODE); graph.setEstimatedEntries(ESTIMATED_ENTRIES); graph.setIdPropertyName("nodeId"); + graph.setEdgeClass("similar"); graph.begin(); } +// @Override +// protected void post() { +// graph.end(); +// } + +// @Override +// protected Long getOrCreate(String value) +// { +// final long v = Long.parseLong(value); +// graph.createVertex(v); +// return v; +// } + +// @Override +// protected void relateNodes(Long src, Long dest) +// { +// graph.createEdge(src, dest); +// } + + + @Override - protected void post() { - graph.end(); + protected void relateNodes(Vertex src, Vertex dest) + { + orientGraph.addEdge(null, src, dest, "similar"); + + // TODO why commit twice? is this a nested transaction? + if (orientGraph instanceof TransactionalGraph) + { + orientGraph.commit(); + } } @Override - protected Long getOrCreate(String value) + protected Vertex getOrCreate(final String value) { - final long v = Long.parseLong(value); - graph.createVertex(v); + final int key = Integer.parseInt(value); + + Vertex v; + final OIdentifiable rec = (OIdentifiable) index.get(key); + if (rec != null) + { + return orientGraph.getVertex(rec); + } + + v = orientGraph.addVertex(key, "nodeId", key); + + + if (orientGraph instanceof TransactionalGraph) + { + orientGraph.commit(); + } + return v; } @Override - protected void relateNodes(Long src, Long dest) + protected void post() { - graph.createEdge(src, dest); + super.post(); + if (orientGraph instanceof TransactionalGraph) + { + orientGraph.commit(); + } } } diff --git a/src/main/java/eu/socialsensor/insert/OrientSingleInsertion.java b/src/main/java/eu/socialsensor/insert/OrientSingleInsertion.java index 39d4875..31dbaec 100644 --- a/src/main/java/eu/socialsensor/insert/OrientSingleInsertion.java +++ b/src/main/java/eu/socialsensor/insert/OrientSingleInsertion.java @@ -8,6 +8,7 @@ import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraph; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import eu.socialsensor.main.GraphDatabaseType; /** @@ -38,7 +39,6 @@ protected void relateNodes(Vertex src, Vertex dest) if (orientGraph instanceof TransactionalGraph) { orientGraph.commit(); - orientGraph.commit(); } } @@ -56,6 +56,7 @@ protected Vertex getOrCreate(final String value) v = orientGraph.addVertex(key, "nodeId", key); + if (orientGraph instanceof TransactionalGraph) { orientGraph.commit(); diff --git a/src/test/java/eu/socialsensor/main/GraphDatabaseBenchmarkTest.java b/src/test/java/eu/socialsensor/main/GraphDatabaseBenchmarkTest.java index b00d00e..608ee68 100644 --- a/src/test/java/eu/socialsensor/main/GraphDatabaseBenchmarkTest.java +++ b/src/test/java/eu/socialsensor/main/GraphDatabaseBenchmarkTest.java @@ -3,6 +3,7 @@ import static org.junit.Assert.fail; import org.junit.Test; + public class GraphDatabaseBenchmarkTest { @Test diff --git a/src/test/resources/META-INF/input.properties b/src/test/resources/META-INF/input.properties index 7c271b5..5f90750 100644 --- a/src/test/resources/META-INF/input.properties +++ b/src/test/resources/META-INF/input.properties @@ -1,11 +1,11 @@ # Choose which data sets you want to include in the benchmark by removing the contents. -#eu.socialsensor.dataset=data/Email-Enron.txt +eu.socialsensor.dataset=data/Email-Enron.txt #eu.socialsensor.dataset=data/com-youtube.ungraph.txt #eu.socialsensor.dataset=data/Amazon0601.txt #eu.socialsensor.dataset=data/com-lj.ungraph.txt #can change the number in the filename of the synthetic datasets to 1000, 5000, 10000, 20000, 30000, 40000, 50000 -eu.socialsensor.dataset=data/network1000.dat -eu.socialsensor.actual-communities=data/community1000.dat +#eu.socialsensor.dataset=data/network1000.dat +#eu.socialsensor.actual-communities=data/community1000.dat eu.socialsensor.database-storage-directory=storage # Sample meters this frequently (milliseconds) @@ -17,14 +17,14 @@ eu.socialsensor.metrics.csv.directory=metrics # Choose which databases you want to in the benchmark by removing the comments. # Available dbs are: -eu.socialsensor.databases=tbdb -eu.socialsensor.databases=tddb +#eu.socialsensor.databases=tbdb +#eu.socialsensor.databases=tddb #eu.socialsensor.databases=tc #eu.socialsensor.databases=thb #eu.socialsensor.databases=tce #eu.socialsensor.databases=tp -#eu.socialsensor.databases=orient -#eu.socialsensor.databases=neo4j +eu.socialsensor.databases=orient +eu.socialsensor.databases=neo4j #eu.socialsensor.databases=sparksee # Database specific options @@ -75,10 +75,10 @@ eu.socialsensor.permute-benchmarks=false # Choose which benchmark you want to run by removing the comments. Choose one Insertion # workload and then query/clustering workloads afterward. -eu.socialsensor.benchmarks=MASSIVE_INSERTION +#eu.socialsensor.benchmarks=MASSIVE_INSERTION #eu.socialsensor.benchmarks=SINGLE_INSERTION -eu.socialsensor.benchmarks=FIND_NEIGHBOURS -eu.socialsensor.benchmarks=FIND_ADJACENT_NODES +#eu.socialsensor.benchmarks=FIND_NEIGHBOURS +#eu.socialsensor.benchmarks=FIND_ADJACENT_NODES eu.socialsensor.benchmarks=FIND_SHORTEST_PATH eu.socialsensor.shortest-path-random-nodes=100