diff --git a/benchmark.ttl b/benchmark.ttl index 1725da4c..eb4861c0 100644 --- a/benchmark.ttl +++ b/benchmark.ttl @@ -22,6 +22,7 @@ hobbit:hasParameter :seed, :numberOfNodes, + :numberOfGraphs, :averageTriplesPerNode, :blankNodesRatio, :literalsRatio, @@ -84,6 +85,7 @@ hobbit:hasParameter :seed, :numberOfNodes, + :numberOfGraphs, :averageTriplesPerNode, :blankNodesRatio, :literalsRatio, @@ -144,6 +146,13 @@ rdfs:domain hobbit:Experiment, hobbit:Challenge; rdfs:range xsd:integer; hobbit:defaultValue "10"^^xsd:integer . + +:numberOfGraphs a hobbit:Parameter, hobbit:ConfigurableParameter; + rdfs:label "Number of graphs"@en; + rdfs:comment "The number of graphs a node can have."@en; + rdfs:domain hobbit:Experiment, hobbit:Challenge; + rdfs:range xsd:integer; + hobbit:defaultValue "1"^^xsd:integer . :averageTriplesPerNode a hobbit:Parameter, hobbit:ConfigurableParameter; rdfs:label "RDF dataset size"@en; diff --git a/docs/README.md b/docs/README.md index 3f8f73cf..c5e3776f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -41,6 +41,7 @@ For the full license text, see [LICENSE](../LICENSE). | Parameter | Description | Ontology resources | --- | --- | --- | Number of nodes | The number of nodes in the synthetic graph. | [orca:numberOfNodes](http://w3id.org/dice-research/orca/ontology#numberOfNodes) +| Number of graphs | The number of graphs a node can have. | [orca:numberOfGraphs](http://w3id.org/dice-research/orca/ontology#numberOfGraphs) | Average node degree | The average degree of the nodes in the generated graph. | [orca:averageNodeGraphDegree](http://w3id.org/dice-research/orca/ontology#averageNodeGraphDegree) | Ratio of blank nodes | Factor that is multiplied to the number of nodes to determine the number of blank nodes that are added to the graph. | [orca:blankNodesRatio](http://w3id.org/dice-research/orca/ontology#blankNodesRatio) | Ratio of literals | Factor that is multiplied to the number of nodes to determine the number of literals that are added to the graph. | [orca:literalsRatio](http://w3id.org/dice-research/orca/ontology#literalsRatio) diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/ApiConstants.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/ApiConstants.java index 7391fab5..b55e88bd 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/ApiConstants.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/ApiConstants.java @@ -14,6 +14,7 @@ public class ApiConstants { public static final String ENV_CRAWL_DELAY_KEY = "LDCBENCH_CRAWL_DELAY"; public static final String ENV_DISALLOWED_RATIO_KEY = "LDCBENCH_DISALLOWED_RATIO"; public static final String ENV_COMPRESSED_RATIO_KEY = "LDCBENCH_COMPRESSED_RATIO"; + public static final String ENV_NUMBER_OF_GRAPHS_KEY = "LDCBENCH_DATAGENERATOR_NUMBER_OF_GRAPHS"; public static final String ENV_SEED_KEY = "LDCBENCH_DATAGENERATOR_SEED"; public static final String ENV_COMPONENT_COUNT_KEY = "LDCBENCH_COMPONENT_COUNT"; diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/Graph.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/Graph.java index 47bb8eae..36f61a72 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/Graph.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/Graph.java @@ -119,6 +119,8 @@ public default int incomingEdgeCount(int nodeId) { * the ID of the external graph the node belongs to. */ public int getGraphId(int nodeId); + + public int getGraphId(); /** * Returns the ID of the node in the external graph if this node is an external diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphBuilder.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphBuilder.java index 963f0220..142eac9f 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphBuilder.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphBuilder.java @@ -3,7 +3,7 @@ /** * An extension of the {@link Graph} interface offering methods for the addition * of nodes and edges. - * + * * @author Michael Röder (michael.roeder@uni-paderborn.de) * */ @@ -13,7 +13,7 @@ public interface GraphBuilder extends Graph { * Returns a {@link Graph} instance based on the graph data that the builder * collected until now. This instance might offer a faster implementation than * the builder class. - * + * * @return a {@link Graph} instance containing the built graph. */ public default Graph build() { @@ -23,7 +23,7 @@ public default Graph build() { /** * Adds a new edge with the given information if possible. Returns {@code true} * if the edge could be added. - * + * * @param sourceId * the id of the source node * @param targetId @@ -37,7 +37,7 @@ public default Graph build() { /** * Adds a new node to the graph and returns its ID. - * + * * @return the ID of the newly created node */ public int addNode(); @@ -45,7 +45,7 @@ public default Graph build() { /** * Adds the given number of nodes to the graph and returns the range of node * IDs. - * + * * @param nodeCount * the number of nodes that should be added * @return the range of IDs from the first added node ([0], inclusive) to the @@ -56,7 +56,7 @@ public default Graph build() { /** * Sets the IDs of the entrance nodes (i.e., nodes from which all other nodes * can be reached). - * + * * @param entranceNodes * the IDs of the entrance nodes */ @@ -68,7 +68,7 @@ public default Graph build() { * graph. However, to mark external nodes (i.e., nodes that are not part of this * graph but are linked from one of the nodes of this graph), this method can be * used to store their graph ID as well as their ID in the other graph. - * + * * @param nodeId * the ID of the node in this graph * @param graphId @@ -80,7 +80,8 @@ public default Graph build() { * graphID != {@link #INTERNAL_NODE_GRAPH_ID}) */ public void setGraphIdOfNode(int nodeId, int graphId, int externalId); - + + public void setGraphId(int i); /** * Sets the range of the blank nodes of the graph. * diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphMetadata.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphMetadata.java index bb302981..b1519c4d 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphMetadata.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphMetadata.java @@ -8,4 +8,5 @@ public class GraphMetadata implements Serializable { public int numberOfNodes; public int[] entranceNodes; + public int graphId; } diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java index a8bad2af..6673f813 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java @@ -39,6 +39,10 @@ public class GrphBasedGraph implements GraphBuilder { /** * Index of first blank node of this graph. */ + protected int blankNodesIndex = Integer.MAX_VALUE; + + protected int actualGraphId; + protected int[] blankNodesRange = {Integer.MAX_VALUE,Integer.MAX_VALUE}; /** * Index of first literal of this graph. @@ -117,7 +121,7 @@ public int[] outgoingEdgeTargets(int nodeId) { public int[] outgoingEdgeTypes(int nodeId) { return orderedOutgoingEdges(nodeId).map(this::getEdgeType).toArray(); } - + @Override public int outgoingEdgeCount(int nodeId) { return graph.getOutEdgeDegree(nodeId); @@ -130,7 +134,7 @@ public int[] incomingEdgeSources(int nodeId) { public int[] incomingEdgeTypes(int nodeId) { return orderedIncomingEdges(nodeId).map(this::getEdgeType).toArray(); } - + @Override public int incomingEdgeCount(int nodeId) { return graph.getInEdgeDegree(nodeId); @@ -279,6 +283,14 @@ public String toString() { return graph.toString(); } + public int getGraphId() { + return actualGraphId; + } + + public void setGraphId(int id) { + actualGraphId = id; + } + @Override public boolean isBlankNode(int nodeId) { return nodeId >= getBlankNodesRange()[0] && nodeId < getBlankNodesRange()[1]; diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/vocab/LDCBench.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/vocab/LDCBench.java index f72e7a33..1e1a049d 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/vocab/LDCBench.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/vocab/LDCBench.java @@ -31,6 +31,7 @@ public class LDCBench { public static final Property httpDumpNodeCompressedRatio = property("httpDumpNodeCompressedRatio"); public static final Property httpDumpNodeWeight = property("httpDumpNodeWeight"); public static final Property numberOfNodes = property("numberOfNodes"); + public static final Property numberOfGraphs = property("numberOfGraphs"); public static final Property blankNodesRatio = property("blankNodesRatio"); public static final Property literalsRatio = property("literalsRatio"); public static final Property numberOfNodesLinks = property("numberOfNodesLinks"); diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/BenchmarkController.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/BenchmarkController.java index b4d18b12..b111abb4 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/BenchmarkController.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/BenchmarkController.java @@ -232,6 +232,8 @@ public void init() throws Exception { .getInt(); int averageRdfGraphDegree = RdfHelper.getLiteral(benchmarkParamModel, null, LDCBench.averageRdfGraphDegree) .getInt(); + int numberOfGraphs = RdfHelper.getLiteral(benchmarkParamModel, null, LDCBench.numberOfGraphs) + .getInt(); double httpDumpNodeCompressedRatio = RdfHelper .getLiteral(benchmarkParamModel, null, LDCBench.httpDumpNodeCompressedRatio).getDouble(); @@ -446,7 +448,7 @@ public void handleNodeGraph(int senderId, Graph g) { DataGenerator.ENV_ACCESS_URI_TEMPLATES_KEY + "=" + serializedAccessUris, DataGenerator.ENV_RESOURCE_URI_TEMPLATES_KEY + "=" + serializedResourceUris, }, nodeManagers.get(i).getDataGeneratorEnvironment(averageRdfGraphDegree, - nodeSizeDeterminer.getNodeSize())); + nodeSizeDeterminer.getNodeSize(), numberOfGraphs)); createDataGenerator(nodeManagers.get(i), envVariables); // FIXME: HOBBIT SDK workaround (setting environment for "containers") if (sdk) { diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/CkanNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/CkanNodeManager.java index e9d84ab4..e0d28c45 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/CkanNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/CkanNodeManager.java @@ -16,10 +16,11 @@ public boolean canBeHub() { } @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + 0, DataGenerator.ENV_NUMBER_OF_NODES_KEY + "=" + 1, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + 1, }; return env; } diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/DereferencingHttpNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/DereferencingHttpNodeManager.java index 5ef83b2d..0588ca01 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/DereferencingHttpNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/DereferencingHttpNodeManager.java @@ -32,10 +32,11 @@ public String[] getNodeEnvironment() { }; @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + averageRdfGraphDegree, DataGenerator.ENV_NUMBER_OF_EDGES_KEY + "=" + triplesPerNode, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + 1, }; return env; } diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/HttpDumpNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/HttpDumpNodeManager.java index d7e43ad7..c59b481f 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/HttpDumpNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/HttpDumpNodeManager.java @@ -23,10 +23,11 @@ public String[] getNodeEnvironment() { }; @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + averageRdfGraphDegree, DataGenerator.ENV_NUMBER_OF_EDGES_KEY + "=" + triplesPerNode, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + numberOfGraphs, }; return env; } diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/JsonLdNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/JsonLdNodeManager.java index 81f597fb..826f9b6b 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/JsonLdNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/JsonLdNodeManager.java @@ -29,10 +29,11 @@ public String[] getNodeEnvironment() { }; @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + averageRdfGraphDegree, DataGenerator.ENV_NUMBER_OF_EDGES_KEY + "=" + triplesPerNode, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + 1, }; return env; } diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/MicrodataNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/MicrodataNodeManager.java index 8588d23a..a1c1ddd1 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/MicrodataNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/MicrodataNodeManager.java @@ -29,10 +29,11 @@ public String[] getNodeEnvironment() { }; @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + 0, DataGenerator.ENV_NUMBER_OF_NODES_KEY + "=" + 1, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + 1, }; return env; } diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/NodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/NodeManager.java index 27b6fda2..ca20d27f 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/NodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/NodeManager.java @@ -6,7 +6,7 @@ * Interface of a manager class that gives the benchmark controller the * necessary information to create a node of a certain type and supply it with * the given data and meta data. - * + * * @author Michael Röder (michael.roeder@uni-paderborn.de) * */ @@ -15,7 +15,7 @@ public interface NodeManager { /** * Flag indicating whether the node created by this node manager can be used as * a hub - * + * * @return flag indicating whether the node created by this node manager can be * used as a hub */ @@ -23,21 +23,21 @@ public interface NodeManager { /** * Returns the Docker image name of the node. - * + * * @return the Docker image name of the node. */ public String getNodeImageName(); /** * Returns the environmental values for the generated node. - * + * * @return the environmental values for the generated node. */ public String[] getNodeEnvironment(); /** * Returns the Docker image name of the node's data generator. - * + * * @return the Docker image name of the node's data generator. */ public String getDataGeneratorImageName(); @@ -45,24 +45,24 @@ public interface NodeManager { /** * Returns the environmental values for the generated node's data generator * based on the given data. - * + * * @param averageRdfGraphDegree the average degree of a node within the * generated RDF graph * @param triplesPerNode the average number of triples per node * @return the environmental values for the generated node's data generator. */ - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode); + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs); /** * Returns the label of the node type. - * + * * @return the label of the node type. */ public abstract String getLabel(); /** * Returns the metadata of this node. - * + * * @return the metadata of this node. */ public NodeMetadata getMetadata(); @@ -70,7 +70,7 @@ public interface NodeManager { /** * Returns the weight for a link from the given node manager instance to this * node manager instance. - * + * * @param nodeManager the node manager instance that could create the outgoing link * @return the weight of such a link in the range of [0,1] */ diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/RDFaNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/RDFaNodeManager.java index 98a7eeda..94163b38 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/RDFaNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/RDFaNodeManager.java @@ -29,10 +29,11 @@ public String[] getNodeEnvironment() { }; @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + 0, DataGenerator.ENV_NUMBER_OF_NODES_KEY + "=" + 1, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + 1, }; return env; } diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/SparqlNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/SparqlNodeManager.java index 636e3bed..f5ec1b50 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/SparqlNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/SparqlNodeManager.java @@ -16,10 +16,11 @@ public boolean canBeHub() { } @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + averageRdfGraphDegree, DataGenerator.ENV_NUMBER_OF_EDGES_KEY + "=" + triplesPerNode, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + 1, }; return env; } diff --git a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java index 072783a2..224e406e 100644 --- a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java +++ b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java @@ -45,13 +45,14 @@ * Data generator of ORCA. It fulfills two main roles: either, it generates the * node graph, i.e., the graph comprising the single nodes (= servers) of the * synthetic cloud, or it generates the RDF graph for a single node. - * + * * @author Michael Röder (michael.roeder@uni-paderborn.de) * */ public class DataGenerator extends AbstractDataGenerator { public static final String ENV_TYPE_KEY = "LDCBENCH_DATAGENERATOR_TYPE"; public static final String ENV_NUMBER_OF_NODES_KEY = "LDCBENCH_DATAGENERATOR_NUMBER_OF_NODES"; + public static final String ENV_NUMBER_OF_GRAPHS_KEY = "LDCBENCH_DATAGENERATOR_NUMBER_OF_GRAPHS"; public static final String ENV_AVERAGE_DEGREE_KEY = "LDCBENCH_DATAGENERATOR_AVERAGE_DEGREE"; public static final String ENV_BLANK_NODES_RATIO ="LDCBENCH_DATAGENERATOR_BLANK_NODES_RATIO"; public static final String ENV_LITERALS_RATIO ="LDCBENCH_DATAGENERATOR_LITERALS_RATIO"; @@ -68,7 +69,7 @@ public class DataGenerator extends AbstractDataGenerator { /** * Types of data generator instances. - * + * * @author Michael Röder (michael.roeder@uni-paderborn.de) */ public static enum Types { @@ -221,6 +222,7 @@ protected void addInterlinks(GraphBuilder g, int numberOfnodesLinks) { for (Map.Entry entry : rdfMetadata.entrySet()) { int targetNodeGraph = entry.getKey(); GraphMetadata gm = entry.getValue(); + if (gm.entranceNodes.length == 0) { throw new IllegalStateException("Node " + getNodeId() + " needs to link to node " + targetNodeGraph + " but there are no entrypoints."); @@ -232,6 +234,7 @@ protected void addInterlinks(GraphBuilder g, int numberOfnodesLinks) { // add a new node int externalNode = g.addNode(); g.setGraphIdOfNode(externalNode, targetNodeGraph, entranceNode); + g.setGraphId(gm.graphId); int propertyId = i; g.addEdge(nodeWithOutgoingLink, externalNode, propertyId); LOGGER.debug("Added the edge ({}, {}, {}) where the target is node {} in graph {}.", nodeWithOutgoingLink, @@ -304,7 +307,7 @@ protected void sendFinalGraph(Graph g) throws Exception { /** * Factory method to create the {@link GraphGenerator} instance used to * generated the node graph. - * + * * @return the {@link GraphGenerator} instance used to generated the node graph */ protected GraphGenerator createNodeGraphGenerator() { @@ -320,7 +323,7 @@ protected GraphGenerator createNodeGraphGenerator() { /** * Factory method to create the {@link GraphGenerator} instance used to * generated the RDF graph. - * + * * @return the {@link GraphGenerator} instance used to generated the RDF graph */ protected GraphGenerator createRDFGraphGenerator() { @@ -349,6 +352,7 @@ public void init() throws Exception { seed = seedGenerator.getNextSeed(); numberOfNodes = EnvVariables.getInt(ENV_NUMBER_OF_NODES_KEY, 0); + int numberOfGraphs = EnvVariables.getInt(ENV_NUMBER_OF_GRAPHS_KEY, 1); avgDegree = Double.parseDouble(EnvVariables.getString(ENV_AVERAGE_DEGREE_KEY)); numberOfEdges = EnvVariables.getInt(ENV_NUMBER_OF_EDGES_KEY, 0); @@ -384,20 +388,41 @@ public void init() throws Exception { generator = createRDFGraphGenerator(); } - GraphBuilder graph = new GrphBasedGraph(); + //initialize all graphs + GraphBuilder graphs[] = new GraphBuilder[numberOfGraphs]; + for (int i = 0; i < numberOfGraphs; i++) { + graphs[i] = new GrphBasedGraph(); + graphs[i].setGraphId(i); + } if (type == Types.NODE_GRAPH_GENERATOR) { - nodeGraph = graph; + nodeGraph = graphs[0]; } if (numberOfNodes != 0) { - LOGGER.debug("Generator {} : Generating a graph with {} nodes {} average degree and {} seed", generatorId, - numberOfNodes, avgDegree, seed); - generator.generateGraph(numberOfNodes, avgDegree, seed, graph); + LOGGER.debug("Generator {} : Generating {} graphs", generatorId, numberOfGraphs); + for (int i = 0; i < graphs.length; i++) { + LOGGER.debug("Generator {} : Generating a graph with {} nodes {} average degree and {} seed", generatorId, + numberOfNodes, avgDegree, seed); + seed = seedGenerator.getNextSeed(); + generator.generateGraph(numberOfNodes/numberOfGraphs, avgDegree, seed, graphs[i]); + } } else { LOGGER.debug("Generator {} : Generating a graph with {} average degree and {} edges and {} seed", generatorId, avgDegree, numberOfEdges, seed); - generator.generateGraph(avgDegree, numberOfEdges, seed, graph); + for (int i = 0; i < graphs.length; i++) { + generator.generateGraph(avgDegree, numberOfEdges, seed, graphs[i]); + seed = seedGenerator.getNextSeed(); + } + } + + if (type == Types.RDF_GRAPH_GENERATOR) { + seed = seedGenerator.getNextSeed(); + blankNodesRatio = Double.parseDouble(EnvVariables.getString(ENV_BLANK_NODES_RATIO)); + for (int i = 0; i < graphs.length; i++) { + addBlankNodes(graphs[i], (int) Math.ceil(graphs[i].getNumberOfNodes() * blankNodesRatio), + seed); + } } if (type == Types.NODE_GRAPH_GENERATOR) { @@ -421,17 +446,20 @@ public void init() throws Exception { buf.put(data); dataGeneratorsChannel.basicPublish(dataGeneratorsExchange, "", null, buf.array()); } else { - // Broadcast our graph's metadata. - GraphMetadata gm = new GraphMetadata(); - gm.numberOfNodes = graph.getNumberOfNodes(); - gm.entranceNodes = graph.getEntranceNodes(); - - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - buf.write(header.array(), 0, header.capacity()); - ObjectOutputStream output = new ObjectOutputStream(buf); - output.writeObject(gm); - - dataGeneratorsChannel.basicPublish(dataGeneratorsExchange, "", null, buf.toByteArray()); + // Broadcast the graph's metadata for every graph. + for (GraphBuilder mygraph : graphs) { + GraphMetadata gm = new GraphMetadata(); + gm.numberOfNodes = mygraph.getNumberOfNodes(); + gm.entranceNodes = mygraph.getEntranceNodes(); + gm.graphId = mygraph.getGraphId(); + + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + buf.write(header.array(), 0, header.capacity()); + ObjectOutputStream output = new ObjectOutputStream(buf); + output.writeObject(gm); + + dataGeneratorsChannel.basicPublish(dataGeneratorsExchange, "", null, buf.toByteArray()); + } } if (type == Types.RDF_GRAPH_GENERATOR) { @@ -445,9 +473,17 @@ public void init() throws Exception { targetMetadataReceivedSemaphore.acquire(rdfMetadata.size()); LOGGER.info("Got all relevant rdf graphs.", generatorId); +<<<<<<< HEAD +======= int numberOfnodesLinks = Integer.parseInt(EnvVariables.getString(ENV_NUMBER_OF_NODES_LINKS)); addInterlinks(graph, numberOfnodesLinks); +>>>>>>> develop + + for (int i = 0; i < numberOfGraphs; i++) { + addInterlinks(graphs[i]); + } + // Send the final graph(s) data. blankNodesRatio = Double.parseDouble(EnvVariables.getString(ENV_BLANK_NODES_RATIO)); addBlankNodes(graph, (int) Math.ceil(graph.getNumberOfNodes() * blankNodesRatio), seedGenerator.getNextSeed()); @@ -457,7 +493,8 @@ public void init() throws Exception { // Send the final graph data. LOGGER.info("Sending the final rdf graph data..."); - sendFinalGraph(graph); + for(GraphBuilder g: graphs) + sendFinalGraph(g); } LOGGER.debug("Generation done.", generatorId); @@ -466,7 +503,7 @@ public void init() throws Exception { /** * Parses the given string assuming that it has been generated with * {@link Arrays#toString()}. - * + * * @param envValue the string containing the array * @return the parsed String array */ diff --git a/ldcbench.eval-module/src/main/java/org/dice_research/ldcbench/benchmark/EvalModule.java b/ldcbench.eval-module/src/main/java/org/dice_research/ldcbench/benchmark/EvalModule.java index 6a5a1b24..6c89cda0 100644 --- a/ldcbench.eval-module/src/main/java/org/dice_research/ldcbench/benchmark/EvalModule.java +++ b/ldcbench.eval-module/src/main/java/org/dice_research/ldcbench/benchmark/EvalModule.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Date; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Timer; @@ -65,7 +66,7 @@ public class EvalModule extends AbstractPlatformConnectorComponent { protected Semaphore crawlingFinished = new Semaphore(0); protected Semaphore dataGenerationFinished = new Semaphore(0); - protected String graphFiles[]; + protected List graphFiles; private NodeMetadata[] nodeMetadata; protected long startTimeStamp; protected long endTimeStamp; @@ -77,6 +78,8 @@ public class EvalModule extends AbstractPlatformConnectorComponent { public void init() throws Exception { super.init(); + graphFiles = new LinkedList(); + sdk = EnvVariables.getBoolean(ApiConstants.ENV_SDK_KEY, false, LOGGER); // Get the experiment URI @@ -104,7 +107,8 @@ public void handle(NodeMetadata[] body) { public void handle(String[] files) { Arrays.sort(files); LOGGER.debug("Got files: {}", Arrays.toString(files)); - graphFiles = files; + for (String f : files) + graphFiles.add(f); } }; @@ -232,7 +236,7 @@ private void sendResultModel(Model model) throws IOException { private Map runEvaluation() { // Evaluate the results based on the data from the SPARQL storage - TripleBlockStreamSupplier supplier = new FileBasedTripleBlockStreamSupplier(graphFiles, + TripleBlockStreamSupplier supplier = new FileBasedTripleBlockStreamSupplier(graphFiles.toArray(new String[0]), Stream.of(nodeMetadata).map(nm -> nm.getResourceUriTemplate()).toArray(String[]::new), Stream.of(nodeMetadata).map(nm -> nm.getAccessUriTemplate()).toArray(String[]::new), new GraphBasedTripleBlockStreamCreator(), new TTLTarGZBasedTripleBlockStreamCreator()); diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java index 03ff86ed..596c93d6 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java @@ -68,6 +68,7 @@ public class SimpleHttpServerComponent extends NodeComponent implements Componen protected Lang dumpFileLang = null; protected CompressionStreamFactory dumpFileCompression = null; protected Archiver dumpfileArchiver = null; + protected int noOfGraphs; @Override public void initBeforeDataGeneration() throws Exception { @@ -75,6 +76,7 @@ public void initBeforeDataGeneration() throws Exception { compressedRatio = Double.parseDouble(EnvVariables.getString(ApiConstants.ENV_COMPRESSED_RATIO_KEY, LOGGER)); disallowedRatio = Double.parseDouble(EnvVariables.getString(ApiConstants.ENV_DISALLOWED_RATIO_KEY, LOGGER)); crawlDelay = EnvVariables.getInt(ApiConstants.ENV_CRAWL_DELAY_KEY, LOGGER); + noOfGraphs = EnvVariables.getInt(ApiConstants.ENV_NUMBER_OF_GRAPHS_KEY,1,LOGGER); String hostname = InetAddress.getLocalHost().getHostName(); LOGGER.info("Hostname: {}", hostname); @@ -175,12 +177,17 @@ protected Container createContainer() throws Exception { String[] accessUriTemplates = Stream.of(nodeMetadata).map(nm -> nm.getAccessUriTemplate()) .toArray(String[]::new); if (dumpFileNode) { - resource = DumpFileResource.create(cloudNodeId.get(), - Stream.of(nodeMetadata).map(nm -> nm.getResourceUriTemplate()).toArray(String[]::new), - Stream.of(nodeMetadata).map(nm -> nm.getAccessUriTemplate()).toArray(String[]::new), - graphs.toArray(new Graph[graphs.size()]), - r -> r.getPath().toString().equals(dumpFilePath), - dumpFileLang, dumpFileCompression, dumpfileArchiver); + for(int i=0; i r.getPath().toString().equals(dumpFilePath), + dumpFileLang, dumpFileCompression, dumpfileArchiver); + + resources.add(resource); + } } else { SimpleTripleCreator tripleCreator = new SimpleTripleCreator(cloudNodeId.get(), resourceUriTemplates, accessUriTemplates); @@ -225,9 +232,9 @@ protected Container createContainer() throws Exception { && r.getTarget().contains(UriHelper.RESOURCE_NODE_TYPE)), contentTypes.toArray(new String[contentTypes.size()])); resource = graphBasedResource; + resources.add(resource); } Objects.requireNonNull(resource, "Couldn't create crawleable resource. Exiting."); - resources.add(resource); return new CrawleableResourceContainer(resources.toArray(new CrawleableResource[resources.size()])); } diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java index 5dab5599..ea5373c0 100755 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java @@ -63,15 +63,17 @@ public class DumpFileBuilder { protected final Lang lang; protected final CompressionStreamFactory compression; protected File dumpFile; + protected int dumpfileCount; public DumpFileBuilder(int domainId, String[] resourceUriTemplates, String[] accessUriTemplates, Graph[] graphs, - Lang lang, CompressionStreamFactory compression) { + Lang lang, CompressionStreamFactory compression, int dumpfileCount) { this.domainId = domainId; this.resourceUriTemplates = resourceUriTemplates; this.accessUriTemplates = accessUriTemplates; this.graphs = graphs; this.lang = lang; this.compression = compression; + this.dumpfileCount = dumpfileCount; } public File build() @@ -97,7 +99,7 @@ private OutputStream generateOutputStream() // fileNameBuilder.append('.'); // fileNameBuilder.append(fileExt.get(0)); // } - dumpFile = File.createTempFile("ldcbench", ".dump"); + dumpFile = File.createTempFile("ldcbench_" + this.dumpfileCount, ".dump"); OutputStream out = new FileOutputStream(dumpFile); out = new BufferedOutputStream(out); if (compression != null) { @@ -167,7 +169,7 @@ private void writeData(OutputStream out, Lang lang) { TripleIterator iterator; LOGGER.info("Domain ID: " + domainId); LOGGER.info("graph size: " + graphs.length); - + Model model = ModelFactory.createDefaultModel(); org.apache.jena.graph.Graph modelGraph = model.getGraph(); int datasetId = 0; diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java index c2577baa..7056a61a 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java @@ -35,17 +35,16 @@ public class DumpFileResource extends AbstractCrawleableResource { new ZipArchiver()); public static DumpFileResource create(int domainId, String[] resourceUriTemplates, String[] accessUriTemplates, - Graph[] graphs, Predicate predicate, Lang lang, CompressionStreamFactory compression, Archiver archiver) { + Graph[] graphs, int dumpfileCount, Predicate predicate, Lang lang, CompressionStreamFactory compression, Archiver archiver) { DumpFileBuilder builder = new DumpFileBuilder(domainId, resourceUriTemplates, accessUriTemplates, graphs, - lang, compression); + lang, compression, dumpfileCount); try { File dumpFile = builder.build(); String contentType = builder.buildContentType(); if (archiver != null) { - //TODO support more than one file //Add dump Files to a List and put them into Archive File archive = File.createTempFile("ldcbench", ".archive"); - archiver.buildArchive(archive,dumpFile); + archiver.buildArchive(archive, dumpFile); return new DumpFileResource(predicate, contentType, archive); } return new DumpFileResource(predicate, contentType, dumpFile); diff --git a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java index 9206baae..9bd22d63 100644 --- a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java +++ b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java @@ -59,14 +59,16 @@ public DumpFileBuilderTest(int domainId, Graph[] graphs, Lang lang, CompressionS @Test public void test() throws NoSuchMethodException, SecurityException, IOException, ReflectiveOperationException, ParserException, NotFoundException { + int dumpfileCount = 0; //only create a single file DumpFileBuilder builder = new DumpFileBuilder(domainId, RESOURCE_URI_TEMPLATES, ACCESS_URI_TEMPLATES, graphs, - lang, compression); + lang, compression, dumpfileCount); System.out.println( "Testing " + lang + (compression == null ? "" : (" with compression " + compression.getMediaType()))); File file = builder.build(); + Model writtenModel = ModelFactory.createDefaultModel(); - + if(!lang.equals(LangUtils.HDT_LANG)) { try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(file))) { RDFParser parser = RDFParser.create().forceLang(lang).source(in).build(); diff --git a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java index 21304801..1de86216 100644 --- a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java +++ b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java @@ -10,6 +10,8 @@ import java.io.InputStream; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -61,6 +63,19 @@ public void test() throws IOException { File archive = File.createTempFile("dummy", ".archive"); archiver.buildArchive(archive, fileToArchive); + //archive multiple files + Path path = Files.createTempDirectory("multiDump-"); + File file1 = File.createTempFile("tempFile1", ".dump",path.toFile()); + FileUtils.writeByteArrayToFile(file1, data); + File file2 = File.createTempFile("tempFile2", ".dump",path.toFile()); + FileUtils.writeByteArrayToFile(file2, data); + File file3 = File.createTempFile("tempFile3", ".dump",path.toFile()); + FileUtils.writeByteArrayToFile(file3, data); + File multiArchive = File.createTempFile("multiDump", ".archive"); + archiver.buildArchive(multiArchive, path.toFile()); + + + //unarchive FileInputStream fis = new FileInputStream(archive); ArchiveInputStream ais; @@ -74,8 +89,33 @@ public void test() throws IOException { // TODO Auto-generated catch block e.printStackTrace(); } + + //unarchive multiple files + FileInputStream fisMulti = new FileInputStream(multiArchive); + ArchiveInputStream aisMulti; + File unarchivedFile1 = null; + File unarchivedFile2 = null; + File unarchivedFile3 = null; + try { + aisMulti = generateInputStream(new BufferedInputStream(fisMulti), unarchiverClassName, decompressionClassName); + aisMulti.getNextEntry(); + unarchivedFile1 = File.createTempFile("file1", ".unarchived"); + IOUtils.copy(aisMulti, new FileOutputStream(unarchivedFile1)); + aisMulti.getNextEntry(); + unarchivedFile2 = File.createTempFile("file2", ".unarchived"); + IOUtils.copy(aisMulti, new FileOutputStream(unarchivedFile2)); + aisMulti.getNextEntry(); + unarchivedFile3 = File.createTempFile("file3", ".unarchived"); + IOUtils.copy(aisMulti, new FileOutputStream(unarchivedFile3)); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } assertTrue(FileUtils.contentEquals(fileToArchive, unarchivedFile)); + assertTrue(FileUtils.contentEquals(file1, unarchivedFile1)); + assertTrue(FileUtils.contentEquals(file2, unarchivedFile2)); + assertTrue(FileUtils.contentEquals(file3, unarchivedFile3)); } protected static byte[] generateData() { diff --git a/ldcbench.integration-test/integration-benchmark-parameters-lemming.ttl b/ldcbench.integration-test/integration-benchmark-parameters-lemming.ttl index 77f1f3a2..532ea394 100644 --- a/ldcbench.integration-test/integration-benchmark-parameters-lemming.ttl +++ b/ldcbench.integration-test/integration-benchmark-parameters-lemming.ttl @@ -28,6 +28,7 @@ experiments:New a hobbit:Experiment; :ckanNodeWeight 0; :sparqlNodeWeight 0; :numberOfNodes 4; + :numberOfGraphs 2; :blankNodesRatio 0.05; :literalsRatio 0.05; :numberOfNodesLinks 3; diff --git a/ldcbench.integration-test/integration-benchmark-parameters.ttl b/ldcbench.integration-test/integration-benchmark-parameters.ttl index 8daf9f80..62debd0a 100644 --- a/ldcbench.integration-test/integration-benchmark-parameters.ttl +++ b/ldcbench.integration-test/integration-benchmark-parameters.ttl @@ -30,6 +30,7 @@ experiments:New a hobbit:Experiment; :microformatNodeWeight 1; :sparqlNodeWeight 0.01; :numberOfNodes 4; + :numberOfGraphs 2; :averageNodeGraphDegree 2; :blankNodesRatio 0.05; :literalsRatio 0.05; diff --git a/ldcbench.integration-test/test-benchmark-parameters-lemming.ttl b/ldcbench.integration-test/test-benchmark-parameters-lemming.ttl index 11c38a25..334198e9 100644 --- a/ldcbench.integration-test/test-benchmark-parameters-lemming.ttl +++ b/ldcbench.integration-test/test-benchmark-parameters-lemming.ttl @@ -28,6 +28,7 @@ experiments:New a hobbit:Experiment; :ckanNodeWeight 0; :sparqlNodeWeight 0; :numberOfNodes 3; + :numberOfGraphs 2; :blankNodesRatio 0.05; :literalsRatio 0.05; :numberOfNodesLinks 3; diff --git a/ldcbench.integration-test/test-benchmark-parameters.ttl b/ldcbench.integration-test/test-benchmark-parameters.ttl index e075404b..f3304a6d 100644 --- a/ldcbench.integration-test/test-benchmark-parameters.ttl +++ b/ldcbench.integration-test/test-benchmark-parameters.ttl @@ -38,6 +38,7 @@ experiments:New a hobbit:Experiment; # at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:583) :sparqlNodeWeight 0; :numberOfNodes 3; + :numberOfGraphs 2; :averageNodeGraphDegree 2; :blankNodesRatio 0.05; :literalsRatio 0.05; diff --git a/ldcbench.microdata-gen/src/main/java/ldcbench/microdata/gen/MicrodataGenerator.java b/ldcbench.microdata-gen/src/main/java/ldcbench/microdata/gen/MicrodataGenerator.java index 3a87aa5d..18a15166 100644 --- a/ldcbench.microdata-gen/src/main/java/ldcbench/microdata/gen/MicrodataGenerator.java +++ b/ldcbench.microdata-gen/src/main/java/ldcbench/microdata/gen/MicrodataGenerator.java @@ -20,6 +20,7 @@ import org.apache.commons.io.FileUtils; import org.dice_research.ldcbench.ApiConstants; import org.dice_research.ldcbench.benchmark.DataGenerator; +import org.dice_research.ldcbench.rdf.RDFNodeType; import org.dice_research.ldcbench.graph.Graph; import org.dice_research.ldcbench.rdf.SimpleTripleCreator; import org.dice_research.ldcbench.utils.tar.TarFileGenerator; @@ -97,7 +98,7 @@ protected void generateEntranceFile(Graph graph, Map htmlFiles, Fi protected String generateEntranceNodeUri(Graph graph, SimpleTripleCreator creator) { int entranceNode = graph.getEntranceNodes()[0]; return creator - .createNode(entranceNode, graph.getExternalNodeId(entranceNode), graph.getGraphId(entranceNode), false) + .createNode(entranceNode, graph.getExternalNodeId(entranceNode), graph.getGraphId(entranceNode), RDFNodeType.IRI) .getURI(); } @@ -106,7 +107,7 @@ protected Set generateOutgoingLinks(final Graph graph, final SimpleTripl int entranceNodes[] = graph.getEntranceNodes(); for (int i = 0; i < entranceNodes.length; ++i) { outgoingLinks.addAll(Arrays.stream(graph.outgoingEdgeTargets(entranceNodes[i])) - .mapToObj(id -> creator.createNode(id, graph.getExternalNodeId(id), graph.getGraphId(id), false)) + .mapToObj(id -> creator.createNode(id, graph.getExternalNodeId(id), graph.getGraphId(id), graph.getNodeType(id))) .map(n -> n.getURI()).collect(Collectors.toSet())); } return outgoingLinks; diff --git a/ldcbench.microformat-gen/src/main/java/ldcbench/microformat/gen/MicroformatGenerator.java b/ldcbench.microformat-gen/src/main/java/ldcbench/microformat/gen/MicroformatGenerator.java index bc7e9fad..435a60d0 100644 --- a/ldcbench.microformat-gen/src/main/java/ldcbench/microformat/gen/MicroformatGenerator.java +++ b/ldcbench.microformat-gen/src/main/java/ldcbench/microformat/gen/MicroformatGenerator.java @@ -30,6 +30,7 @@ import org.dice_research.ldcbench.benchmark.DataGenerator; import org.dice_research.ldcbench.graph.Graph; import org.dice_research.ldcbench.microformat.gen.MicroformatEntranceFileGenerator; +import org.dice_research.ldcbench.rdf.RDFNodeType; import org.dice_research.ldcbench.rdf.SimpleTripleCreator; import org.dice_research.ldcbench.utils.tar.TarFileGenerator; import org.hobbit.core.rabbit.RabbitQueueFactory; @@ -111,7 +112,7 @@ protected void generateEntranceFile(Graph graph, Map htmlFiles, Fi protected String generateEntranceNodeUri(Graph graph, SimpleTripleCreator creator) { int entranceNode = graph.getEntranceNodes()[0]; return creator - .createNode(entranceNode, graph.getExternalNodeId(entranceNode), graph.getGraphId(entranceNode), false) + .createNode(entranceNode, graph.getExternalNodeId(entranceNode), graph.getGraphId(entranceNode), RDFNodeType.IRI) .getURI(); } @@ -120,7 +121,7 @@ protected Set generateOutgoingLinks(final Graph graph, final SimpleTripl int entranceNodes[] = graph.getEntranceNodes(); for (int i = 0; i < entranceNodes.length; ++i) { outgoingLinks.addAll(Arrays.stream(graph.outgoingEdgeTargets(entranceNodes[i])) - .mapToObj(id -> creator.createNode(id, graph.getExternalNodeId(id), graph.getGraphId(id), false)) + .mapToObj(id -> creator.createNode(id, graph.getExternalNodeId(id), graph.getGraphId(id), graph.getNodeType(id))) .map(n -> n.getURI()).collect(Collectors.toSet())); } return outgoingLinks; diff --git a/ldcbench.nodes/src/main/java/org/dice_research/ldcbench/nodes/rabbit/GraphHandler.java b/ldcbench.nodes/src/main/java/org/dice_research/ldcbench/nodes/rabbit/GraphHandler.java index fceaba74..5a298509 100644 --- a/ldcbench.nodes/src/main/java/org/dice_research/ldcbench/nodes/rabbit/GraphHandler.java +++ b/ldcbench.nodes/src/main/java/org/dice_research/ldcbench/nodes/rabbit/GraphHandler.java @@ -26,8 +26,9 @@ public void run() { try { super.run(); if(!encounteredError()) { - // FIXME fix usage of file [0] - handleData(Files.readAllBytes(new File(receivedFiles[0]).toPath())); + for (String myFiles : receivedFiles) { + handleData(Files.readAllBytes(new File(myFiles).toPath())); + } } } catch (Exception e) { LOGGER.error("Error while reading graph. Increasing error count.", e);