From 851cf3f4e24f8b63c9ab4851283fad4b3bee88a0 Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Sun, 31 Oct 2021 23:50:17 +0100 Subject: [PATCH 01/27] Created list of dumpfiles --- .../nodes/http/simple/dump/DumpFileResource.java | 3 +++ .../nodes/http/simple/dump/DumpFileBuilderTest.java | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java index e1443320..6629a756 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.function.Predicate; @@ -34,6 +35,7 @@ public class DumpFileResource extends AbstractCrawleableResource { public static final List ARCHIVERS = Arrays.asList(new TarArchiver(), new TarArchiver(ReflectionBasedStreamFactory.create("java.util.zip.GZIPOutputStream", "application/gzip", ".gz")), new ZipArchiver()); + public static final ArrayList multipleFile = new ArrayList(); public static DumpFileResource create(int domainId, String[] resourceUriTemplates, String[] accessUriTemplates, Graph[] graphs, Predicate predicate, Lang lang, CompressionStreamFactory compression, Archiver archiver) { @@ -42,6 +44,7 @@ public static DumpFileResource create(int domainId, String[] resourceUriTemplate try { File dumpFile = builder.build(); String contentType = builder.buildContentType(); + multipleFile.add(dumpFile); if (archiver != null) { //TODO support more than one file //Add dump Files to a List and put them into Archive diff --git a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java index 0ca42116..4567e37c 100644 --- a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java +++ b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java @@ -5,6 +5,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.Set; @@ -41,6 +42,7 @@ public class DumpFileBuilderTest { private Lang lang; private CompressionStreamFactory compression; private Model expectedModel; + public static final ArrayList multipleFile = new ArrayList(); public DumpFileBuilderTest(int domainId, Graph[] graphs, Lang lang, CompressionStreamFactory compression, Model expectedModel) { @@ -59,7 +61,14 @@ public void test() throws NoSuchMethodException, SecurityException, IOException, "Testing " + lang + (compression == null ? "" : (" with compression " + compression.getMediaType()))); File file = builder.build(); - + multipleFile.add(file); + + Iterator it = multipleFile.iterator(); + + while(it.hasNext()) { + System.out.print(it.next() + " "); + } + Model writtenModel = ModelFactory.createDefaultModel(); try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(file))) { RDFParser parser = RDFParser.create().forceLang(lang).source(in).build(); From e1807ad16af847206ed4fef3426cd5e48483ea61 Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Sun, 7 Nov 2021 21:47:45 +0100 Subject: [PATCH 02/27] Archiving the dumpfile list --- .../nodes/http/simple/dump/DumpFileResource.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java index 6629a756..e6ff7a7f 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java @@ -8,9 +8,11 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.function.Predicate; +import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.io.IOUtils; import org.apache.jena.riot.Lang; import org.dice_research.ldcbench.graph.Graph; @@ -48,8 +50,13 @@ public static DumpFileResource create(int domainId, String[] resourceUriTemplate if (archiver != null) { //TODO support more than one file //Add dump Files to a List and put them into Archive + Iterator it = multipleFile.iterator(); File archive = File.createTempFile("ldcbench", ".archive"); - archiver.buildArchive(archive,dumpFile); + ArchiveOutputStream aos = archiver.createStream(archive); + while(it.hasNext()) { + archiver.addFileToArchive(aos, it.next()); + } + //archiver.buildArchive(archive,dumpFile); contentType = archiver.getMediaType(); return new DumpFileResource(predicate, contentType, archive); } From eda6e67211c2eaab11092aa626a4c35d6b2b5d18 Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Thu, 11 Nov 2021 23:18:23 +0100 Subject: [PATCH 03/27] Testing MultiDump --- .../http/simple/dump/DumpFileBuilderTest.java | 8 ---- .../http/simple/dump/comp/ArchiverTest.java | 37 +++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java index 4567e37c..a9a8820b 100644 --- a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java +++ b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java @@ -5,7 +5,6 @@ import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.Set; @@ -61,13 +60,6 @@ public void test() throws NoSuchMethodException, SecurityException, IOException, "Testing " + lang + (compression == null ? "" : (" with compression " + compression.getMediaType()))); File file = builder.build(); - multipleFile.add(file); - - Iterator it = multipleFile.iterator(); - - while(it.hasNext()) { - System.out.print(it.next() + " "); - } Model writtenModel = ModelFactory.createDefaultModel(); try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(file))) { diff --git a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java index 21304801..67d1c531 100644 --- a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java +++ b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java @@ -15,6 +15,7 @@ import java.util.Random; import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.junit.Test; @@ -61,6 +62,19 @@ public void test() throws IOException { File archive = File.createTempFile("dummy", ".archive"); archiver.buildArchive(archive, fileToArchive); + //archive multi + File file1 = File.createTempFile("dummy1", ".dump"); + FileUtils.writeByteArrayToFile(file1, data); + File file2 = File.createTempFile("dummy2", ".dump"); + FileUtils.writeByteArrayToFile(file2, data); + File file3 = File.createTempFile("dummy3", ".dump"); + FileUtils.writeByteArrayToFile(file3, data); + File multiArchive = File.createTempFile("multiDump", ".archive"); + ArchiveOutputStream aos = archiver.createStream(multiArchive); + archiver.addFileToArchive(aos, file1); + archiver.addFileToArchive(aos, file2); + archiver.addFileToArchive(aos, file3); + //unarchive FileInputStream fis = new FileInputStream(archive); ArchiveInputStream ais; @@ -74,8 +88,31 @@ public void test() throws IOException { // TODO Auto-generated catch block e.printStackTrace(); } + + //unarchive multi + FileInputStream fisMulti = new FileInputStream(multiArchive); + ArchiveInputStream aisMulti; + File unarchivedFile1 = null; + File unarchivedFile2 = null; + File unarchivedFile3 = null; + try { + aisMulti = generateInputStream(new BufferedInputStream(fisMulti), unarchiverClassName, decompressionClassName); + aisMulti.getNextEntry(); + unarchivedFile1 = File.createTempFile("file1", ".unarchived"); + IOUtils.copy(aisMulti, new FileOutputStream(unarchivedFile1)); + unarchivedFile2 = File.createTempFile("file2", ".unarchived"); + IOUtils.copy(aisMulti, new FileOutputStream(unarchivedFile2)); + unarchivedFile3 = File.createTempFile("file3", ".unarchived"); + IOUtils.copy(aisMulti, new FileOutputStream(unarchivedFile3)); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } assertTrue(FileUtils.contentEquals(fileToArchive, unarchivedFile)); + assertTrue(FileUtils.contentEquals(file1, unarchivedFile1)); + assertTrue(FileUtils.contentEquals(file2, unarchivedFile2)); + assertTrue(FileUtils.contentEquals(file3, unarchivedFile3)); } protected static byte[] generateData() { From ff2cbd1d8d497b1353bdf9416755aa080dd7a4cd Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Fri, 12 Nov 2021 16:26:29 +0100 Subject: [PATCH 04/27] TempDir approach for archiving --- .../http/simple/dump/comp/ArchiverTest.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java index 67d1c531..7bb99b3d 100644 --- a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java +++ b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java @@ -10,12 +10,13 @@ import java.io.InputStream; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Random; import org.apache.commons.compress.archivers.ArchiveInputStream; -import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.junit.Test; @@ -57,26 +58,27 @@ public void test() throws IOException { byte[] data = generateData(); //archive - File fileToArchive = File.createTempFile("dummy", ".dump"); + /*File fileToArchive = File.createTempFile("dummy", ".dump"); FileUtils.writeByteArrayToFile(fileToArchive, data); File archive = File.createTempFile("dummy", ".archive"); - archiver.buildArchive(archive, fileToArchive); + archiver.buildArchive(archive, fileToArchive);*/ //archive multi - File file1 = File.createTempFile("dummy1", ".dump"); + + Path path = Files.createTempDirectory("multiDump-"); + File file1 = File.createTempFile("tempFile1", ".dump",path.toFile()); FileUtils.writeByteArrayToFile(file1, data); - File file2 = File.createTempFile("dummy2", ".dump"); + File file2 = File.createTempFile("tempFile2", ".dump",path.toFile()); FileUtils.writeByteArrayToFile(file2, data); - File file3 = File.createTempFile("dummy3", ".dump"); + File file3 = File.createTempFile("tempFile3", ".dump",path.toFile()); FileUtils.writeByteArrayToFile(file3, data); File multiArchive = File.createTempFile("multiDump", ".archive"); - ArchiveOutputStream aos = archiver.createStream(multiArchive); - archiver.addFileToArchive(aos, file1); - archiver.addFileToArchive(aos, file2); - archiver.addFileToArchive(aos, file3); + archiver.buildArchive(multiArchive, path.toFile()); + + //unarchive - FileInputStream fis = new FileInputStream(archive); + /*FileInputStream fis = new FileInputStream(archive); ArchiveInputStream ais; File unarchivedFile = null; try { @@ -87,7 +89,7 @@ public void test() throws IOException { } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); - } + }*/ //unarchive multi FileInputStream fisMulti = new FileInputStream(multiArchive); @@ -109,7 +111,7 @@ public void test() throws IOException { e.printStackTrace(); } - assertTrue(FileUtils.contentEquals(fileToArchive, unarchivedFile)); + //assertTrue(FileUtils.contentEquals(fileToArchive, unarchivedFile)); assertTrue(FileUtils.contentEquals(file1, unarchivedFile1)); assertTrue(FileUtils.contentEquals(file2, unarchivedFile2)); assertTrue(FileUtils.contentEquals(file3, unarchivedFile3)); From c95c3f8a46df9d45b4c3ac7c2dc9237b55c485cd Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Fri, 12 Nov 2021 16:33:35 +0100 Subject: [PATCH 05/27] Fixed the test cases --- .../http/simple/dump/comp/ArchiverTest.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java index 7bb99b3d..1de86216 100644 --- a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java +++ b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/comp/ArchiverTest.java @@ -58,13 +58,12 @@ public void test() throws IOException { byte[] data = generateData(); //archive - /*File fileToArchive = File.createTempFile("dummy", ".dump"); + File fileToArchive = File.createTempFile("dummy", ".dump"); FileUtils.writeByteArrayToFile(fileToArchive, data); File archive = File.createTempFile("dummy", ".archive"); - archiver.buildArchive(archive, fileToArchive);*/ - - //archive multi + archiver.buildArchive(archive, fileToArchive); + //archive multiple files Path path = Files.createTempDirectory("multiDump-"); File file1 = File.createTempFile("tempFile1", ".dump",path.toFile()); FileUtils.writeByteArrayToFile(file1, data); @@ -78,7 +77,7 @@ public void test() throws IOException { //unarchive - /*FileInputStream fis = new FileInputStream(archive); + FileInputStream fis = new FileInputStream(archive); ArchiveInputStream ais; File unarchivedFile = null; try { @@ -89,9 +88,9 @@ public void test() throws IOException { } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); - }*/ + } - //unarchive multi + //unarchive multiple files FileInputStream fisMulti = new FileInputStream(multiArchive); ArchiveInputStream aisMulti; File unarchivedFile1 = null; @@ -102,8 +101,10 @@ public void test() throws IOException { aisMulti.getNextEntry(); unarchivedFile1 = File.createTempFile("file1", ".unarchived"); IOUtils.copy(aisMulti, new FileOutputStream(unarchivedFile1)); + aisMulti.getNextEntry(); unarchivedFile2 = File.createTempFile("file2", ".unarchived"); IOUtils.copy(aisMulti, new FileOutputStream(unarchivedFile2)); + aisMulti.getNextEntry(); unarchivedFile3 = File.createTempFile("file3", ".unarchived"); IOUtils.copy(aisMulti, new FileOutputStream(unarchivedFile3)); } catch (Exception e) { @@ -111,7 +112,7 @@ public void test() throws IOException { e.printStackTrace(); } - //assertTrue(FileUtils.contentEquals(fileToArchive, unarchivedFile)); + assertTrue(FileUtils.contentEquals(fileToArchive, unarchivedFile)); assertTrue(FileUtils.contentEquals(file1, unarchivedFile1)); assertTrue(FileUtils.contentEquals(file2, unarchivedFile2)); assertTrue(FileUtils.contentEquals(file3, unarchivedFile3)); From 0e2c156c53854cabeefbf6cf5ba7023a216c7c02 Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Sun, 14 Nov 2021 22:03:19 +0100 Subject: [PATCH 06/27] Added boolean multiplefile param for DumpfileBuilder --- .../simple/SimpleHttpServerComponent.java | 2 +- .../http/simple/dump/DumpFileBuilder.java | 23 +++++++++++++++++-- .../http/simple/dump/DumpFileResource.java | 4 ++-- .../http/simple/dump/DumpFileBuilderTest.java | 11 +++++++-- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java index 7d40abeb..2cf7dd12 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java @@ -179,7 +179,7 @@ protected Container createContainer() throws Exception { Stream.of(nodeMetadata).map(nm -> nm.getAccessUriTemplate()).toArray(String[]::new), graphs.toArray(new Graph[graphs.size()]), r -> r.getPath().toString().equals(dumpFilePath), - dumpFileLang, dumpFileCompression, dumpfileArchiver); + dumpFileLang, dumpFileCompression, false,dumpfileArchiver); } else { SimpleTripleCreator tripleCreator = new SimpleTripleCreator(cloudNodeId.get(), resourceUriTemplates, accessUriTemplates); diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java index 6adfb6ec..03f9d329 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java @@ -7,6 +7,8 @@ import java.io.IOException; import java.io.OutputStream; import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.List; @@ -56,15 +58,20 @@ public class DumpFileBuilder { protected final Lang lang; protected final CompressionStreamFactory compression; protected File dumpFile; + protected final boolean multipleDump; + protected Path path; public DumpFileBuilder(int domainId, String[] resourceUriTemplates, String[] accessUriTemplates, Graph[] graphs, - Lang lang, CompressionStreamFactory compression) { + Lang lang, CompressionStreamFactory compression,boolean multipleDump) { this.domainId = domainId; this.resourceUriTemplates = resourceUriTemplates; this.accessUriTemplates = accessUriTemplates; this.graphs = graphs; this.lang = lang; this.compression = compression; + this.multipleDump = multipleDump; + this.generateFolder(); + } public File build() @@ -86,7 +93,10 @@ private OutputStream generateOutputStream() // fileNameBuilder.append('.'); // fileNameBuilder.append(fileExt.get(0)); // } - dumpFile = File.createTempFile("ldcbench", ".dump"); + if(multipleDump==true) { + dumpFile = File.createTempFile("ldcbench", ".dump",path.toFile()); + }else + dumpFile = File.createTempFile("ldcbench", ".dump"); OutputStream out = new FileOutputStream(dumpFile); out = new BufferedOutputStream(out); if (compression != null) { @@ -156,6 +166,15 @@ public String buildContentType() { return compression.getMediaType(); } return lang.getHeaderString(); + } + + private void generateFolder() { + try { + path = Files.createTempDirectory("multiDump-"); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } } diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java index e6ff7a7f..04e610c5 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java @@ -40,9 +40,9 @@ public class DumpFileResource extends AbstractCrawleableResource { public static final ArrayList multipleFile = new ArrayList(); public static DumpFileResource create(int domainId, String[] resourceUriTemplates, String[] accessUriTemplates, - Graph[] graphs, Predicate predicate, Lang lang, CompressionStreamFactory compression, Archiver archiver) { + Graph[] graphs, Predicate predicate, Lang lang, CompressionStreamFactory compression, boolean multipleDump, Archiver archiver) { DumpFileBuilder builder = new DumpFileBuilder(domainId, resourceUriTemplates, accessUriTemplates, graphs, - lang, compression); + lang, compression,multipleDump); try { File dumpFile = builder.build(); String contentType = builder.buildContentType(); diff --git a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java index a9a8820b..2cbb1880 100644 --- a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java +++ b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java @@ -4,6 +4,8 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -41,7 +43,7 @@ public class DumpFileBuilderTest { private Lang lang; private CompressionStreamFactory compression; private Model expectedModel; - public static final ArrayList multipleFile = new ArrayList(); + private boolean multipleDump; public DumpFileBuilderTest(int domainId, Graph[] graphs, Lang lang, CompressionStreamFactory compression, Model expectedModel) { @@ -55,12 +57,17 @@ public DumpFileBuilderTest(int domainId, Graph[] graphs, Lang lang, CompressionS @Test public void test() throws NoSuchMethodException, SecurityException, IOException, ReflectiveOperationException { DumpFileBuilder builder = new DumpFileBuilder(domainId, RESOURCE_URI_TEMPLATES, ACCESS_URI_TEMPLATES, graphs, - lang, compression); + lang, compression,true); System.out.println( "Testing " + lang + (compression == null ? "" : (" with compression " + compression.getMediaType()))); File file = builder.build(); + if(builder.multipleDump) { + File file2 = builder.build(); + File file3 = builder.build(); + } + Model writtenModel = ModelFactory.createDefaultModel(); try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(file))) { RDFParser parser = RDFParser.create().forceLang(lang).source(in).build(); From 9b7105a191620d7a36170d6b4c64a33bce8cbf73 Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Wed, 17 Nov 2021 23:44:14 +0100 Subject: [PATCH 07/27] Trying to split the graph --- .../ldcbench/graph/GrphBasedGraph.java | 28 +++++++++++++++++++ .../ldcbench/graph/GrphBasedGraphTest.java | 21 ++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java index 366bbaae..e6fbb609 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java @@ -59,6 +59,34 @@ public GrphBasedGraph(Graph other) { } setEntranceNodes(other.getEntranceNodes()); } + + public GrphBasedGraph[] SplitGraph(Graph graph) { + int nodes = graph.getNumberOfNodes(); +// int count; +// count = graph.getEntranceNodes().length; + GrphBasedGraph[] splitGraph = new GrphBasedGraph[2]; + for (int i = 0; i< nodes/2; i++) { + int[] targets = graph.outgoingEdgeTargets(i); + int[] types = graph.outgoingEdgeTypes(i); + for (int j = 0; j < targets.length; j++) { + splitGraph[0].addEdge(i, targets[j], types[j]); + } + if (graph.getGraphId(i) != Graph.INTERNAL_NODE_GRAPH_ID) { + setGraphIdOfNode(i, graph.getGraphId(i), graph.getExternalNodeId(i)); + } + } + for (int i = nodes/2; i< nodes/2; i++) { + int[] targets = graph.outgoingEdgeTargets(i); + int[] types = graph.outgoingEdgeTypes(i); + for (int j = 0; j < targets.length; j++) { + splitGraph[1].addEdge(i, targets[j], types[j]); + } + if (graph.getGraphId(i) != Graph.INTERNAL_NODE_GRAPH_ID) { + setGraphIdOfNode(i, graph.getGraphId(i), graph.getExternalNodeId(i)); + } + } + return splitGraph; + } /** * Given an edge ID, returns type of that edge. diff --git a/ldcbench.api/src/test/java/org/dice_research/ldcbench/graph/GrphBasedGraphTest.java b/ldcbench.api/src/test/java/org/dice_research/ldcbench/graph/GrphBasedGraphTest.java index b50be238..f8cff33f 100644 --- a/ldcbench.api/src/test/java/org/dice_research/ldcbench/graph/GrphBasedGraphTest.java +++ b/ldcbench.api/src/test/java/org/dice_research/ldcbench/graph/GrphBasedGraphTest.java @@ -143,4 +143,25 @@ public void testCloningConstructor() { assertEquals("External node id", 3, g2.getExternalNodeId(n2)); assertArrayEquals("Entrance nodes", new int[]{n1, n3}, g2.getEntranceNodes()); } + @Test + public void splitGraph() { + int n1 = g.addNode(); + int n2 = g.addNode(); + int n3 = g.addNode(); + int n4 = g.addNode(); + int n5 = g.addNode(); + int n6 = g.addNode(); + int n7 = g.addNode(); + g.addEdge(n1, n2, 0); + g.addEdge(n2, n3, 1); + g.addEdge(n3, n4, 2); + g.addEdge(n4, n5, 3); + g.addEdge(n5, n6, 4); + g.addEdge(n6, n7, 5); + + g.setEntranceNodes(new int[]{n1, n5}); + GrphBasedGraph[] splitGraph = SplitGraph(); + + + } } From 7e3e6f45e9c437877b68a3a684c13aa6e019cbeb Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Sun, 21 Nov 2021 20:34:57 +0100 Subject: [PATCH 08/27] trying to create multiple graphs in data generator instead of splitting graph --- .../ldcbench/graph/GrphBasedGraph.java | 28 ---------------- .../ldcbench/graph/GrphBasedGraphTest.java | 21 ------------ .../ldcbench/benchmark/DataGenerator.java | 32 ++++++++++++++++++- 3 files changed, 31 insertions(+), 50 deletions(-) diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java index e6fbb609..366bbaae 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java @@ -59,34 +59,6 @@ public GrphBasedGraph(Graph other) { } setEntranceNodes(other.getEntranceNodes()); } - - public GrphBasedGraph[] SplitGraph(Graph graph) { - int nodes = graph.getNumberOfNodes(); -// int count; -// count = graph.getEntranceNodes().length; - GrphBasedGraph[] splitGraph = new GrphBasedGraph[2]; - for (int i = 0; i< nodes/2; i++) { - int[] targets = graph.outgoingEdgeTargets(i); - int[] types = graph.outgoingEdgeTypes(i); - for (int j = 0; j < targets.length; j++) { - splitGraph[0].addEdge(i, targets[j], types[j]); - } - if (graph.getGraphId(i) != Graph.INTERNAL_NODE_GRAPH_ID) { - setGraphIdOfNode(i, graph.getGraphId(i), graph.getExternalNodeId(i)); - } - } - for (int i = nodes/2; i< nodes/2; i++) { - int[] targets = graph.outgoingEdgeTargets(i); - int[] types = graph.outgoingEdgeTypes(i); - for (int j = 0; j < targets.length; j++) { - splitGraph[1].addEdge(i, targets[j], types[j]); - } - if (graph.getGraphId(i) != Graph.INTERNAL_NODE_GRAPH_ID) { - setGraphIdOfNode(i, graph.getGraphId(i), graph.getExternalNodeId(i)); - } - } - return splitGraph; - } /** * Given an edge ID, returns type of that edge. diff --git a/ldcbench.api/src/test/java/org/dice_research/ldcbench/graph/GrphBasedGraphTest.java b/ldcbench.api/src/test/java/org/dice_research/ldcbench/graph/GrphBasedGraphTest.java index f8cff33f..b50be238 100644 --- a/ldcbench.api/src/test/java/org/dice_research/ldcbench/graph/GrphBasedGraphTest.java +++ b/ldcbench.api/src/test/java/org/dice_research/ldcbench/graph/GrphBasedGraphTest.java @@ -143,25 +143,4 @@ public void testCloningConstructor() { assertEquals("External node id", 3, g2.getExternalNodeId(n2)); assertArrayEquals("Entrance nodes", new int[]{n1, n3}, g2.getEntranceNodes()); } - @Test - public void splitGraph() { - int n1 = g.addNode(); - int n2 = g.addNode(); - int n3 = g.addNode(); - int n4 = g.addNode(); - int n5 = g.addNode(); - int n6 = g.addNode(); - int n7 = g.addNode(); - g.addEdge(n1, n2, 0); - g.addEdge(n2, n3, 1); - g.addEdge(n3, n4, 2); - g.addEdge(n4, n5, 3); - g.addEdge(n5, n6, 4); - g.addEdge(n6, n7, 5); - - g.setEntranceNodes(new int[]{n1, n5}); - GrphBasedGraph[] splitGraph = SplitGraph(); - - - } } diff --git a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java index 0e5f5166..b8678bc1 100644 --- a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java +++ b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java @@ -225,6 +225,26 @@ protected void addInterlinks(GraphBuilder g) { } } + protected void sendFinalGraphs(Graph[] g) throws Exception { + for(Graph myg : g) { //should i overide serializer as well? + byte[] data = SerializationHelper.serialize(SERIALIZER_CLASS, myg); + String name = String.format("graph-%0" + (int) Math.ceil(Math.log10(getNumberOfGenerators() + 1)) + "d" + + ApiConstants.FILE_ENDING_GRAPH, getNodeId()); + + // TODO: Use RabbitMQ exchange to send the data (SimpleFileSender doesn't + // support that) + try (InputStream is = new ByteArrayInputStream(data); + SimpleFileSender dataSender = SimpleFileSender.create(outgoingDataQueuefactory, dataQueueName);) { + dataSender.streamData(is, name); + } + + try (InputStream is = new ByteArrayInputStream(data); + SimpleFileSender dataSender = SimpleFileSender.create(outgoingDataQueuefactory, evalDataQueueName);) { + dataSender.streamData(is, name); + } + } + } + protected void sendFinalGraph(Graph g) throws Exception { byte[] data = SerializationHelper.serialize(SERIALIZER_CLASS, g); String name = String.format("graph-%0" + (int) Math.ceil(Math.log10(getNumberOfGenerators() + 1)) + "d" @@ -322,6 +342,12 @@ public void init() throws Exception { } GraphBuilder graph = new GrphBasedGraph(); + + //initialize two graphs + GraphBuilder multiGraph[] = new GraphBuilder[2]; + for (GraphBuilder mygraph : multiGraph) { + mygraph = new GrphBasedGraph(); + } if (type == Types.NODE_GRAPH_GENERATOR) { nodeGraph = graph; @@ -330,7 +356,10 @@ public void init() throws Exception { if (numberOfNodes != 0) { LOGGER.debug("Generator {} : Generating a graph with {} nodes {} average degree and {} seed", generatorId, numberOfNodes, avgDegree, seed); - generator.generateGraph(numberOfNodes, avgDegree, seed, graph); + //generator.generateGraph(numberOfNodes, avgDegree, seed, graph); + for (GraphBuilder mygraph : multiGraph) { + generator.generateGraph(numberOfNodes/2, avgDegree, seed, mygraph); //by 2 because split + } } else { LOGGER.debug("Generator {} : Generating a graph with {} average degree and {} edges and {} seed", generatorId, avgDegree, numberOfEdges, seed); @@ -387,6 +416,7 @@ public void init() throws Exception { // Send the final graph data. LOGGER.info("Sending the final rdf graph data..."); sendFinalGraph(graph); + sendFinalGraphs(multiGraph); } LOGGER.debug("Generation done.", generatorId); From e597e7bca4df904625fdaf1d859d8fb415cd9827 Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Sun, 28 Nov 2021 15:31:39 +0100 Subject: [PATCH 09/27] Graph Handler for multiple files --- .../dice_research/ldcbench/nodes/rabbit/GraphHandler.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ldcbench.nodes/src/main/java/org/dice_research/ldcbench/nodes/rabbit/GraphHandler.java b/ldcbench.nodes/src/main/java/org/dice_research/ldcbench/nodes/rabbit/GraphHandler.java index fceaba74..6b7ae2fd 100644 --- a/ldcbench.nodes/src/main/java/org/dice_research/ldcbench/nodes/rabbit/GraphHandler.java +++ b/ldcbench.nodes/src/main/java/org/dice_research/ldcbench/nodes/rabbit/GraphHandler.java @@ -27,7 +27,13 @@ public void run() { super.run(); if(!encounteredError()) { // FIXME fix usage of file [0] - handleData(Files.readAllBytes(new File(receivedFiles[0]).toPath())); + if(receivedFiles.length==1)//if only one file.. then old implementation + handleData(Files.readAllBytes(new File(receivedFiles[0]).toPath())); + else { //handle multiple files. + for (String myfiles : receivedFiles) { + handleData(Files.readAllBytes(new File(myfiles).toPath())); + } + } } } catch (Exception e) { LOGGER.error("Error while reading graph. Increasing error count.", e); From 6685b2ccd23e5d5d90f0968ecfb8a34c5656f295 Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Sun, 28 Nov 2021 22:45:34 +0100 Subject: [PATCH 10/27] Changed graph properties --- .../dice_research/ldcbench/graph/Graph.java | 2 ++ .../ldcbench/graph/GraphMetadata.java | 1 + .../ldcbench/graph/GrphBasedGraph.java | 6 ++++ .../ldcbench/benchmark/DataGenerator.java | 34 +++++++++++++------ 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/Graph.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/Graph.java index aa4c001f..4f7bdafa 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/Graph.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/Graph.java @@ -117,6 +117,8 @@ public default int incomingEdgeCount(int nodeId) { * the ID of the external graph the node belongs to. */ public int getGraphId(int nodeId); + + public int getGraphId(); /** * Returns the ID of the node in the external graph if this node is an external diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphMetadata.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphMetadata.java index bb302981..b1519c4d 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphMetadata.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphMetadata.java @@ -8,4 +8,5 @@ public class GraphMetadata implements Serializable { public int numberOfNodes; public int[] entranceNodes; + public int graphId; } diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java index 366bbaae..ee515ee5 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java @@ -34,6 +34,8 @@ public class GrphBasedGraph implements GraphBuilder { * external nodes. */ protected Map externalNodes = new HashMap<>(); + + protected int actualGraphId; //from where can we set this? /** * Constructor for an empty GraphBuilder. @@ -253,4 +255,8 @@ public void setGraphIdOfNode(int nodeId, int graphId, int externalId) { public String toString() { return graph.toString(); } + + public int getGraphId() { + return actualGraphId; + } } diff --git a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java index b8678bc1..26ad4af8 100644 --- a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java +++ b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java @@ -388,16 +388,30 @@ public void init() throws Exception { dataGeneratorsChannel.basicPublish(dataGeneratorsExchange, "", null, buf.array()); } else { // Broadcast our graph's metadata. - GraphMetadata gm = new GraphMetadata(); - gm.numberOfNodes = graph.getNumberOfNodes(); - gm.entranceNodes = graph.getEntranceNodes(); - - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - buf.write(header.array(), 0, header.capacity()); - ObjectOutputStream output = new ObjectOutputStream(buf); - output.writeObject(gm); - - dataGeneratorsChannel.basicPublish(dataGeneratorsExchange, "", null, buf.toByteArray()); +// GraphMetadata gm = new GraphMetadata(); +// gm.numberOfNodes = graph.getNumberOfNodes(); +// gm.entranceNodes = graph.getEntranceNodes(); +// +// ByteArrayOutputStream buf = new ByteArrayOutputStream(); +// buf.write(header.array(), 0, header.capacity()); +// ObjectOutputStream output = new ObjectOutputStream(buf); +// output.writeObject(gm); +// +// dataGeneratorsChannel.basicPublish(dataGeneratorsExchange, "", null, buf.toByteArray()); + + for (GraphBuilder mygraph : multiGraph) { + GraphMetadata gm = new GraphMetadata(); + gm.numberOfNodes = mygraph.getNumberOfNodes(); + gm.entranceNodes = mygraph.getEntranceNodes(); + gm.graphId = mygraph.getGraphId(); + + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + buf.write(header.array(), 0, header.capacity()); + ObjectOutputStream output = new ObjectOutputStream(buf); + output.writeObject(gm); + + dataGeneratorsChannel.basicPublish(dataGeneratorsExchange, "", null, buf.toByteArray()); + } } if (type == Types.RDF_GRAPH_GENERATOR) { From 2d79f93cc4c1b1bfc4d28e71708c0146ba34cf74 Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Fri, 3 Dec 2021 19:52:08 +0100 Subject: [PATCH 11/27] Added noOfGraphs as a parameter --- benchmark.ttl | 9 +++++++++ docs/README.md | 1 + .../java/org/dice_research/ldcbench/vocab/LDCBench.java | 1 + .../dice_research/ldcbench/benchmark/DataGenerator.java | 6 ++++++ 4 files changed, 17 insertions(+) diff --git a/benchmark.ttl b/benchmark.ttl index abfaf027..55ba4996 100644 --- a/benchmark.ttl +++ b/benchmark.ttl @@ -22,6 +22,7 @@ hobbit:hasParameter :seed, :numberOfNodes, + :numberOfGraphs, :averageTriplesPerNode, :averageNodeGraphDegree, :averageRdfGraphDegree, @@ -76,6 +77,7 @@ hobbit:hasParameter :seed, :numberOfNodes, + :numberOfGraphs, :averageTriplesPerNode, :averageNodeGraphDegree, :averageRdfGraphDegree, @@ -128,6 +130,13 @@ rdfs:domain hobbit:Experiment, hobbit:Challenge; rdfs:range xsd:integer; hobbit:defaultValue "10"^^xsd:integer . + +:numberOfGraphs a hobbit:Parameter, hobbit:ConfigurableParameter; + rdfs:label "Number of graphs"@en; + rdfs:comment "The number of graphs a node can have."@en; + rdfs:domain hobbit:Experiment, hobbit:Challenge; + rdfs:range xsd:integer; + hobbit:defaultValue "1"^^xsd:integer . :averageTriplesPerNode a hobbit:Parameter, hobbit:ConfigurableParameter; rdfs:label "RDF dataset size"@en; diff --git a/docs/README.md b/docs/README.md index ad94bba3..5fba2ab2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -41,6 +41,7 @@ For the full license text, see [LICENSE](../LICENSE). | Parameter | Description | Ontology resources | --- | --- | --- | Number of nodes | The number of nodes in the synthetic graph. | [orca:numberOfNodes](http://w3id.org/dice-research/orca/ontology#numberOfNodes) +| Number of graphs | The number of graphs a node can have. | [orca:numberOfGraphs](http://w3id.org/dice-research/orca/ontology#numberOfGraphs) | Average node degree | The average degree of the nodes in the generated graph. | [orca:averageNodeGraphDegree](http://w3id.org/dice-research/orca/ontology#averageNodeGraphDegree) | RDF dataset size | Average number of triples of the generated RDF graphs. | [orca:averageTriplesPerNode](http://w3id.org/dice-research/orca/ontology#averageTriplesPerNode) | Average resource degree | The average degree of the resources in the RDF graphs. | [orca:averageRdfGraphDegree](http://w3id.org/dice-research/orca/ontology#averageRdfGraphDegree) diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/vocab/LDCBench.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/vocab/LDCBench.java index eadb3256..d144faad 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/vocab/LDCBench.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/vocab/LDCBench.java @@ -28,6 +28,7 @@ public class LDCBench { public static final Property httpDumpNodeCompressedRatio = property("httpDumpNodeCompressedRatio"); public static final Property httpDumpNodeWeight = property("httpDumpNodeWeight"); public static final Property numberOfNodes = property("numberOfNodes"); + public static final Property numberOfGraphs = property("numberOfGraphs"); public static final Property macroRecall = property("macroRecall"); public static final Property microRecall = property("microRecall"); public static final Property nodeSizeDeterminer = property("nodeSizeDeterminer"); diff --git a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java index 26ad4af8..f1d1cc81 100644 --- a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java +++ b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java @@ -46,6 +46,7 @@ public class DataGenerator extends AbstractDataGenerator { public static final String ENV_TYPE_KEY = "LDCBENCH_DATAGENERATOR_TYPE"; public static final String ENV_NUMBER_OF_NODES_KEY = "LDCBENCH_DATAGENERATOR_NUMBER_OF_NODES"; + public static final String ENV_NUMBER_OF_GRAPHS_KEY = "LDCBENCH_DATAGENERATOR_NUMBER_OF_GRAPHS"; public static final String ENV_AVERAGE_DEGREE_KEY = "LDCBENCH_DATAGENERATOR_AVERAGE_DEGREE"; public static final String ENV_NUMBER_OF_EDGES_KEY = "LDCBENCH_DATAGENERATOR_NUMBER_OF_EDGES"; public static final String ENV_DATA_QUEUE_KEY = "LDCBENCH_DATA_QUEUE"; @@ -113,6 +114,10 @@ public static enum Types { * The number of nodes that should be generated. */ private int numberOfNodes; + /** + * The number of graphs for a node + */ + private int numberOfGraphs; /** * The average degree of the nodes. */ @@ -306,6 +311,7 @@ public void init() throws Exception { seed = seedGenerator.getNextSeed(); numberOfNodes = EnvVariables.getInt(ENV_NUMBER_OF_NODES_KEY, 0); + numberOfGraphs = EnvVariables.getInt(ENV_NUMBER_OF_GRAPHS_KEY, 1); avgDegree = Double.parseDouble(EnvVariables.getString(ENV_AVERAGE_DEGREE_KEY)); numberOfEdges = EnvVariables.getInt(ENV_NUMBER_OF_EDGES_KEY, 0); From 5f0a1d87884aed3a2d413b862ab535156e4069fb Mon Sep 17 00:00:00 2001 From: Thoren-G Date: Sun, 5 Dec 2021 15:26:17 +0100 Subject: [PATCH 12/27] DataGenerator now uses noOfGraph-Parameter --- .../ldcbench/benchmark/DataGenerator.java | 104 +++++++++++------- 1 file changed, 64 insertions(+), 40 deletions(-) diff --git a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java index f1d1cc81..52c0b5df 100644 --- a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java +++ b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java @@ -39,7 +39,7 @@ * Data generator of ORCA. It fulfills two main roles: either, it generates the * node graph, i.e., the graph comprising the single nodes (= servers) of the * synthetic cloud, or it generates the RDF graph for a single node. - * + * * @author Michael Röder (michael.roeder@uni-paderborn.de) * */ @@ -59,7 +59,7 @@ public class DataGenerator extends AbstractDataGenerator { /** * Types of data generator instances. - * + * * @author Michael Röder (michael.roeder@uni-paderborn.de) */ public static enum Types { @@ -230,26 +230,27 @@ protected void addInterlinks(GraphBuilder g) { } } - protected void sendFinalGraphs(Graph[] g) throws Exception { - for(Graph myg : g) { //should i overide serializer as well? - byte[] data = SerializationHelper.serialize(SERIALIZER_CLASS, myg); - String name = String.format("graph-%0" + (int) Math.ceil(Math.log10(getNumberOfGenerators() + 1)) + "d" - + ApiConstants.FILE_ENDING_GRAPH, getNodeId()); - - // TODO: Use RabbitMQ exchange to send the data (SimpleFileSender doesn't - // support that) - try (InputStream is = new ByteArrayInputStream(data); - SimpleFileSender dataSender = SimpleFileSender.create(outgoingDataQueuefactory, dataQueueName);) { - dataSender.streamData(is, name); - } + //TODO remove + // protected void sendFinalGraphs(Graph[] g) throws Exception { + // for(Graph myg : g) { //should i overide serializer as well? + // byte[] data = SerializationHelper.serialize(SERIALIZER_CLASS, myg); + // String name = String.format("graph-%0" + (int) Math.ceil(Math.log10(getNumberOfGenerators() + 1)) + "d" + // + ApiConstants.FILE_ENDING_GRAPH, getNodeId()); + + // // TODO: Use RabbitMQ exchange to send the data (SimpleFileSender doesn't + // // support that) + // try (InputStream is = new ByteArrayInputStream(data); + // SimpleFileSender dataSender = SimpleFileSender.create(outgoingDataQueuefactory, dataQueueName);) { + // dataSender.streamData(is, name); + // } + + // try (InputStream is = new ByteArrayInputStream(data); + // SimpleFileSender dataSender = SimpleFileSender.create(outgoingDataQueuefactory, evalDataQueueName);) { + // dataSender.streamData(is, name); + // } + // } + // } - try (InputStream is = new ByteArrayInputStream(data); - SimpleFileSender dataSender = SimpleFileSender.create(outgoingDataQueuefactory, evalDataQueueName);) { - dataSender.streamData(is, name); - } - } - } - protected void sendFinalGraph(Graph g) throws Exception { byte[] data = SerializationHelper.serialize(SERIALIZER_CLASS, g); String name = String.format("graph-%0" + (int) Math.ceil(Math.log10(getNumberOfGenerators() + 1)) + "d" @@ -271,7 +272,7 @@ protected void sendFinalGraph(Graph g) throws Exception { /** * Factory method to create the {@link GraphGenerator} instance used to * generated the node graph. - * + * * @return the {@link GraphGenerator} instance used to generated the node graph */ protected GraphGenerator createNodeGraphGenerator() { @@ -287,7 +288,7 @@ protected GraphGenerator createNodeGraphGenerator() { /** * Factory method to create the {@link GraphGenerator} instance used to * generated the RDF graph. - * + * * @return the {@link GraphGenerator} instance used to generated the RDF graph */ protected GraphGenerator createRDFGraphGenerator() { @@ -347,29 +348,31 @@ public void init() throws Exception { generator = createRDFGraphGenerator(); } - GraphBuilder graph = new GrphBasedGraph(); - + // GraphBuilder graph = new GrphBasedGraph(); TODO remove + //initialize two graphs - GraphBuilder multiGraph[] = new GraphBuilder[2]; - for (GraphBuilder mygraph : multiGraph) { + GraphBuilder graphs[] = new GraphBuilder[numberOfGraphs]; + for (GraphBuilder mygraph : graphs) { mygraph = new GrphBasedGraph(); } if (type == Types.NODE_GRAPH_GENERATOR) { - nodeGraph = graph; + nodeGraph = graphs[0]; } if (numberOfNodes != 0) { LOGGER.debug("Generator {} : Generating a graph with {} nodes {} average degree and {} seed", generatorId, numberOfNodes, avgDegree, seed); //generator.generateGraph(numberOfNodes, avgDegree, seed, graph); - for (GraphBuilder mygraph : multiGraph) { - generator.generateGraph(numberOfNodes/2, avgDegree, seed, mygraph); //by 2 because split + for (GraphBuilder mygraph : graphs) { + generator.generateGraph(numberOfNodes/numberOfGraphs, avgDegree, seed, mygraph); //TODO how many nodes should the graph have in case of multiple graphs? } } else { LOGGER.debug("Generator {} : Generating a graph with {} average degree and {} edges and {} seed", generatorId, avgDegree, numberOfEdges, seed); - generator.generateGraph(avgDegree, numberOfEdges, seed, graph); + for (GraphBuilder mygraph : graphs) { + generator.generateGraph(avgDegree, numberOfEdges, seed, mygraph); //TODO how many nodes should the graph have in case of multiple graphs? (see above) + } } if (type == Types.NODE_GRAPH_GENERATOR) { @@ -404,18 +407,18 @@ public void init() throws Exception { // output.writeObject(gm); // // dataGeneratorsChannel.basicPublish(dataGeneratorsExchange, "", null, buf.toByteArray()); - - for (GraphBuilder mygraph : multiGraph) { + + for (GraphBuilder mygraph : graphs) { GraphMetadata gm = new GraphMetadata(); gm.numberOfNodes = mygraph.getNumberOfNodes(); gm.entranceNodes = mygraph.getEntranceNodes(); gm.graphId = mygraph.getGraphId(); - + ByteArrayOutputStream buf = new ByteArrayOutputStream(); buf.write(header.array(), 0, header.capacity()); ObjectOutputStream output = new ObjectOutputStream(buf); output.writeObject(gm); - + dataGeneratorsChannel.basicPublish(dataGeneratorsExchange, "", null, buf.toByteArray()); } } @@ -431,12 +434,33 @@ public void init() throws Exception { targetMetadataReceivedSemaphore.acquire(rdfMetadata.size()); LOGGER.info("Got all relevant rdf graphs.", generatorId); - addInterlinks(graph); - // Send the final graph data. + for (GraphBuilder mygraph : graphs) { + addInterlinks(mygraph); + } + + // Send the final graph(s) data. LOGGER.info("Sending the final rdf graph data..."); - sendFinalGraph(graph); - sendFinalGraphs(multiGraph); + for(GraphBuilder g: graphs) + sendFinalGraph(g); + /** + * (i have now implemented approach 3) + * + * ThorenG + * TODO PROBLEM: + * the sendFinalGraph()-function is overridden by the other dataGenerators (like RDFa) + * With this implementation, the other generators will not work, because sendFinalGraph() is never called + * + * 1st approach: We actually only want DumpFileNodes to have multiple graphs?! + * --> only use sendFinalGraphs() in this case + * but the data generator does not now wether we are generating data for the dumpfilenode or any other node.. + * 2nd approach: Change the sendFinalGraphs-Parameter to an Array + * --> the function must be changed everywhere + * 3rd approach: call sendFinalGraphs in a loop + * (best one, isn't it) + * 4th approach: call sendFinalGraph for the first element of graphs and sendFinalGraphs for the remaining + * mehhh.. that's crap + */ } LOGGER.debug("Generation done.", generatorId); @@ -445,7 +469,7 @@ public void init() throws Exception { /** * Parses the given string assuming that it has been generated with * {@link Arrays#toString()}. - * + * * @param envValue the string containing the array * @return the parsed String array */ From c2dc08c151c161c5abee85d6fbd55ad624f9f5bf Mon Sep 17 00:00:00 2001 From: Thoren-G Date: Sun, 5 Dec 2021 15:50:59 +0100 Subject: [PATCH 13/27] DataGenerator will only generate multiple graphs for the DumpfileNode --- .../benchmark/BenchmarkController.java | 4 +++- .../benchmark/cloud/CkanNodeManager.java | 3 ++- .../cloud/DereferencingHttpNodeManager.java | 3 ++- .../benchmark/cloud/HttpDumpNodeManager.java | 3 ++- .../benchmark/cloud/JsonLdNodeManager.java | 3 ++- .../ldcbench/benchmark/cloud/NodeManager.java | 20 +++++++++---------- .../benchmark/cloud/RDFaNodeManager.java | 3 ++- .../benchmark/cloud/SparqlNodeManager.java | 3 ++- 8 files changed, 25 insertions(+), 17 deletions(-) diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/BenchmarkController.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/BenchmarkController.java index 95b9a154..b0bbfeec 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/BenchmarkController.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/BenchmarkController.java @@ -227,6 +227,8 @@ public void init() throws Exception { .getInt(); int averageRdfGraphDegree = RdfHelper.getLiteral(benchmarkParamModel, null, LDCBench.averageRdfGraphDegree) .getInt(); + int numberOfGraphs = RdfHelper.getLiteral(benchmarkParamModel, null, LDCBench.numberOfGraphs + .getInt(); double httpDumpNodeCompressedRatio = RdfHelper .getLiteral(benchmarkParamModel, null, LDCBench.httpDumpNodeCompressedRatio).getDouble(); @@ -441,7 +443,7 @@ public void handleNodeGraph(int senderId, Graph g) { DataGenerator.ENV_ACCESS_URI_TEMPLATES_KEY + "=" + serializedAccessUris, DataGenerator.ENV_RESOURCE_URI_TEMPLATES_KEY + "=" + serializedResourceUris, }, nodeManagers.get(i).getDataGeneratorEnvironment(averageRdfGraphDegree, - nodeSizeDeterminer.getNodeSize())); + nodeSizeDeterminer.getNodeSize(), numberOfGraphs)); createDataGenerator(nodeManagers.get(i), envVariables); // FIXME: HOBBIT SDK workaround (setting environment for "containers") if (sdk) { diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/CkanNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/CkanNodeManager.java index e9d84ab4..e0d28c45 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/CkanNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/CkanNodeManager.java @@ -16,10 +16,11 @@ public boolean canBeHub() { } @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + 0, DataGenerator.ENV_NUMBER_OF_NODES_KEY + "=" + 1, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + 1, }; return env; } diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/DereferencingHttpNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/DereferencingHttpNodeManager.java index 5ef83b2d..0588ca01 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/DereferencingHttpNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/DereferencingHttpNodeManager.java @@ -32,10 +32,11 @@ public String[] getNodeEnvironment() { }; @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + averageRdfGraphDegree, DataGenerator.ENV_NUMBER_OF_EDGES_KEY + "=" + triplesPerNode, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + 1, }; return env; } diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/HttpDumpNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/HttpDumpNodeManager.java index d7e43ad7..c59b481f 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/HttpDumpNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/HttpDumpNodeManager.java @@ -23,10 +23,11 @@ public String[] getNodeEnvironment() { }; @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + averageRdfGraphDegree, DataGenerator.ENV_NUMBER_OF_EDGES_KEY + "=" + triplesPerNode, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + numberOfGraphs, }; return env; } diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/JsonLdNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/JsonLdNodeManager.java index 81f597fb..826f9b6b 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/JsonLdNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/JsonLdNodeManager.java @@ -29,10 +29,11 @@ public String[] getNodeEnvironment() { }; @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + averageRdfGraphDegree, DataGenerator.ENV_NUMBER_OF_EDGES_KEY + "=" + triplesPerNode, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + 1, }; return env; } diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/NodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/NodeManager.java index 27b6fda2..0902133c 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/NodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/NodeManager.java @@ -6,7 +6,7 @@ * Interface of a manager class that gives the benchmark controller the * necessary information to create a node of a certain type and supply it with * the given data and meta data. - * + * * @author Michael Röder (michael.roeder@uni-paderborn.de) * */ @@ -15,7 +15,7 @@ public interface NodeManager { /** * Flag indicating whether the node created by this node manager can be used as * a hub - * + * * @return flag indicating whether the node created by this node manager can be * used as a hub */ @@ -23,21 +23,21 @@ public interface NodeManager { /** * Returns the Docker image name of the node. - * + * * @return the Docker image name of the node. */ public String getNodeImageName(); /** * Returns the environmental values for the generated node. - * + * * @return the environmental values for the generated node. */ public String[] getNodeEnvironment(); /** * Returns the Docker image name of the node's data generator. - * + * * @return the Docker image name of the node's data generator. */ public String getDataGeneratorImageName(); @@ -45,24 +45,24 @@ public interface NodeManager { /** * Returns the environmental values for the generated node's data generator * based on the given data. - * + * * @param averageRdfGraphDegree the average degree of a node within the * generated RDF graph * @param triplesPerNode the average number of triples per node * @return the environmental values for the generated node's data generator. */ - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode); + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs = 1); /** * Returns the label of the node type. - * + * * @return the label of the node type. */ public abstract String getLabel(); /** * Returns the metadata of this node. - * + * * @return the metadata of this node. */ public NodeMetadata getMetadata(); @@ -70,7 +70,7 @@ public interface NodeManager { /** * Returns the weight for a link from the given node manager instance to this * node manager instance. - * + * * @param nodeManager the node manager instance that could create the outgoing link * @return the weight of such a link in the range of [0,1] */ diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/RDFaNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/RDFaNodeManager.java index 9c31b98c..11f20a08 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/RDFaNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/RDFaNodeManager.java @@ -22,10 +22,11 @@ public boolean canBeHub() { } @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + 0, DataGenerator.ENV_NUMBER_OF_NODES_KEY + "=" + 1, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + 1, }; return env; } diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/SparqlNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/SparqlNodeManager.java index 636e3bed..f5ec1b50 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/SparqlNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/SparqlNodeManager.java @@ -16,10 +16,11 @@ public boolean canBeHub() { } @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + averageRdfGraphDegree, DataGenerator.ENV_NUMBER_OF_EDGES_KEY + "=" + triplesPerNode, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + 1, }; return env; } From 2f18ea342d8fc1f35553c7654ea5ca4ad48affda Mon Sep 17 00:00:00 2001 From: Thoren-G Date: Sun, 5 Dec 2021 16:32:13 +0100 Subject: [PATCH 14/27] minor fixes --- .../dice_research/ldcbench/benchmark/BenchmarkController.java | 2 +- .../org/dice_research/ldcbench/benchmark/cloud/NodeManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/BenchmarkController.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/BenchmarkController.java index b0bbfeec..e90cdb3f 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/BenchmarkController.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/BenchmarkController.java @@ -227,7 +227,7 @@ public void init() throws Exception { .getInt(); int averageRdfGraphDegree = RdfHelper.getLiteral(benchmarkParamModel, null, LDCBench.averageRdfGraphDegree) .getInt(); - int numberOfGraphs = RdfHelper.getLiteral(benchmarkParamModel, null, LDCBench.numberOfGraphs + int numberOfGraphs = RdfHelper.getLiteral(benchmarkParamModel, null, LDCBench.numberOfGraphs) .getInt(); double httpDumpNodeCompressedRatio = RdfHelper .getLiteral(benchmarkParamModel, null, LDCBench.httpDumpNodeCompressedRatio).getDouble(); diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/NodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/NodeManager.java index 0902133c..ca20d27f 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/NodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/NodeManager.java @@ -51,7 +51,7 @@ public interface NodeManager { * @param triplesPerNode the average number of triples per node * @return the environmental values for the generated node's data generator. */ - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs = 1); + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs); /** * Returns the label of the node type. From 17dc37a869fda3230ff9676c1426538755ebc8b0 Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Sat, 11 Dec 2021 16:48:29 +0100 Subject: [PATCH 15/27] Added noOfGraphs in httpcomponent --- .../dice_research/ldcbench/ApiConstants.java | 1 + .../simple/SimpleHttpServerComponent.java | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/ApiConstants.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/ApiConstants.java index 7391fab5..b55e88bd 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/ApiConstants.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/ApiConstants.java @@ -14,6 +14,7 @@ public class ApiConstants { public static final String ENV_CRAWL_DELAY_KEY = "LDCBENCH_CRAWL_DELAY"; public static final String ENV_DISALLOWED_RATIO_KEY = "LDCBENCH_DISALLOWED_RATIO"; public static final String ENV_COMPRESSED_RATIO_KEY = "LDCBENCH_COMPRESSED_RATIO"; + public static final String ENV_NUMBER_OF_GRAPHS_KEY = "LDCBENCH_DATAGENERATOR_NUMBER_OF_GRAPHS"; public static final String ENV_SEED_KEY = "LDCBENCH_DATAGENERATOR_SEED"; public static final String ENV_COMPONENT_COUNT_KEY = "LDCBENCH_COMPONENT_COUNT"; diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java index 2cf7dd12..19bf4094 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java @@ -67,6 +67,7 @@ public class SimpleHttpServerComponent extends NodeComponent implements Componen protected Lang dumpFileLang = null; protected CompressionStreamFactory dumpFileCompression = null; protected Archiver dumpfileArchiver = null; + protected int noOfGraphs; @Override public void initBeforeDataGeneration() throws Exception { @@ -74,6 +75,8 @@ public void initBeforeDataGeneration() throws Exception { compressedRatio = Double.parseDouble(EnvVariables.getString(ApiConstants.ENV_COMPRESSED_RATIO_KEY, LOGGER)); disallowedRatio = Double.parseDouble(EnvVariables.getString(ApiConstants.ENV_DISALLOWED_RATIO_KEY, LOGGER)); crawlDelay = EnvVariables.getInt(ApiConstants.ENV_CRAWL_DELAY_KEY, LOGGER); + noOfGraphs = EnvVariables.getInt(ApiConstants.ENV_NUMBER_OF_GRAPHS_KEY,1,LOGGER); + String hostname = InetAddress.getLocalHost().getHostName(); LOGGER.info("Hostname: {}", hostname); @@ -174,12 +177,16 @@ protected Container createContainer() throws Exception { String[] accessUriTemplates = Stream.of(nodeMetadata).map(nm -> nm.getAccessUriTemplate()) .toArray(String[]::new); if (dumpFileNode) { - resource = DumpFileResource.create(cloudNodeId.get(), - Stream.of(nodeMetadata).map(nm -> nm.getResourceUriTemplate()).toArray(String[]::new), - Stream.of(nodeMetadata).map(nm -> nm.getAccessUriTemplate()).toArray(String[]::new), - graphs.toArray(new Graph[graphs.size()]), - r -> r.getPath().toString().equals(dumpFilePath), - dumpFileLang, dumpFileCompression, false,dumpfileArchiver); + for(int i=0; i nm.getResourceUriTemplate()).toArray(String[]::new), + Stream.of(nodeMetadata).map(nm -> nm.getAccessUriTemplate()).toArray(String[]::new), + graphs.toArray(new Graph[graphs.size()]), + r -> r.getPath().toString().equals(dumpFilePath), + dumpFileLang, dumpFileCompression, false,dumpfileArchiver); + + resources.add(resource); + } } else { SimpleTripleCreator tripleCreator = new SimpleTripleCreator(cloudNodeId.get(), resourceUriTemplates, accessUriTemplates); @@ -224,9 +231,9 @@ protected Container createContainer() throws Exception { && r.getTarget().contains(UriHelper.RESOURCE_NODE_TYPE)), contentTypes.toArray(new String[contentTypes.size()])); resource = graphBasedResource; + resources.add(resource); } Objects.requireNonNull(resource, "Couldn't create crawleable resource. Exiting."); - resources.add(resource); return new CrawleableResourceContainer(resources.toArray(new CrawleableResource[resources.size()])); } From 686690280cf79c534d74765d02e83885537087ae Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Sat, 11 Dec 2021 16:55:39 +0100 Subject: [PATCH 16/27] Removed hardcoding boolean value --- .../nodes/http/simple/SimpleHttpServerComponent.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java index 19bf4094..5c834ce4 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java @@ -68,6 +68,7 @@ public class SimpleHttpServerComponent extends NodeComponent implements Componen protected CompressionStreamFactory dumpFileCompression = null; protected Archiver dumpfileArchiver = null; protected int noOfGraphs; + protected boolean multipleFiles=false; @Override public void initBeforeDataGeneration() throws Exception { @@ -76,7 +77,8 @@ public void initBeforeDataGeneration() throws Exception { disallowedRatio = Double.parseDouble(EnvVariables.getString(ApiConstants.ENV_DISALLOWED_RATIO_KEY, LOGGER)); crawlDelay = EnvVariables.getInt(ApiConstants.ENV_CRAWL_DELAY_KEY, LOGGER); noOfGraphs = EnvVariables.getInt(ApiConstants.ENV_NUMBER_OF_GRAPHS_KEY,1,LOGGER); - + if(noOfGraphs >1) + multipleFiles = true; String hostname = InetAddress.getLocalHost().getHostName(); LOGGER.info("Hostname: {}", hostname); @@ -183,7 +185,7 @@ protected Container createContainer() throws Exception { Stream.of(nodeMetadata).map(nm -> nm.getAccessUriTemplate()).toArray(String[]::new), graphs.toArray(new Graph[graphs.size()]), r -> r.getPath().toString().equals(dumpFilePath), - dumpFileLang, dumpFileCompression, false,dumpfileArchiver); + dumpFileLang, dumpFileCompression,multipleFiles,dumpfileArchiver); resources.add(resource); } From 0eac1a779747baf6c42e8a662da039813e383b28 Mon Sep 17 00:00:00 2001 From: Thoren-G Date: Sun, 12 Dec 2021 16:10:35 +0100 Subject: [PATCH 17/27] Fixed the integration tests. The DataGenerator was not able to build any Graphs due to a wrong array initialization --- .../benchmark/cloud/MicrodataNodeManager.java | 3 +- .../ldcbench/benchmark/DataGenerator.java | 32 ++++--------------- 2 files changed, 9 insertions(+), 26 deletions(-) diff --git a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/MicrodataNodeManager.java b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/MicrodataNodeManager.java index 8588d23a..a1c1ddd1 100644 --- a/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/MicrodataNodeManager.java +++ b/ldcbench.controller/src/main/java/org/dice_research/ldcbench/benchmark/cloud/MicrodataNodeManager.java @@ -29,10 +29,11 @@ public String[] getNodeEnvironment() { }; @Override - public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode) { + public String[] getDataGeneratorEnvironment(long averageRdfGraphDegree, long triplesPerNode, long numberOfGraphs) { String[] env = new String[]{ DataGenerator.ENV_AVERAGE_DEGREE_KEY + "=" + 0, DataGenerator.ENV_NUMBER_OF_NODES_KEY + "=" + 1, + DataGenerator.ENV_NUMBER_OF_GRAPHS_KEY + "=" + 1, }; return env; } diff --git a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java index 52c0b5df..58bf827c 100644 --- a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java +++ b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java @@ -352,8 +352,8 @@ public void init() throws Exception { //initialize two graphs GraphBuilder graphs[] = new GraphBuilder[numberOfGraphs]; - for (GraphBuilder mygraph : graphs) { - mygraph = new GrphBasedGraph(); + for (int i = 0; i < numberOfGraphs; i++) { + graphs[i] = new GrphBasedGraph(); } if (type == Types.NODE_GRAPH_GENERATOR) { @@ -361,17 +361,17 @@ public void init() throws Exception { } if (numberOfNodes != 0) { - LOGGER.debug("Generator {} : Generating a graph with {} nodes {} average degree and {} seed", generatorId, - numberOfNodes, avgDegree, seed); - //generator.generateGraph(numberOfNodes, avgDegree, seed, graph); + LOGGER.debug("Generator {} : Generating {} Graphs", generatorId, numberOfGraphs); for (GraphBuilder mygraph : graphs) { - generator.generateGraph(numberOfNodes/numberOfGraphs, avgDegree, seed, mygraph); //TODO how many nodes should the graph have in case of multiple graphs? + LOGGER.debug("Generator {} : Generating a graph with {} nodes {} average degree and {} seed", generatorId, + numberOfNodes, avgDegree, seed); + generator.generateGraph(numberOfNodes/numberOfGraphs, avgDegree, seed, mygraph); } } else { LOGGER.debug("Generator {} : Generating a graph with {} average degree and {} edges and {} seed", generatorId, avgDegree, numberOfEdges, seed); for (GraphBuilder mygraph : graphs) { - generator.generateGraph(avgDegree, numberOfEdges, seed, mygraph); //TODO how many nodes should the graph have in case of multiple graphs? (see above) + generator.generateGraph(avgDegree, numberOfEdges, seed, mygraph); } } @@ -443,24 +443,6 @@ public void init() throws Exception { LOGGER.info("Sending the final rdf graph data..."); for(GraphBuilder g: graphs) sendFinalGraph(g); - /** - * (i have now implemented approach 3) - * - * ThorenG - * TODO PROBLEM: - * the sendFinalGraph()-function is overridden by the other dataGenerators (like RDFa) - * With this implementation, the other generators will not work, because sendFinalGraph() is never called - * - * 1st approach: We actually only want DumpFileNodes to have multiple graphs?! - * --> only use sendFinalGraphs() in this case - * but the data generator does not now wether we are generating data for the dumpfilenode or any other node.. - * 2nd approach: Change the sendFinalGraphs-Parameter to an Array - * --> the function must be changed everywhere - * 3rd approach: call sendFinalGraphs in a loop - * (best one, isn't it) - * 4th approach: call sendFinalGraph for the first element of graphs and sendFinalGraphs for the remaining - * mehhh.. that's crap - */ } LOGGER.debug("Generation done.", generatorId); From 059b34a4a3b616bbe0e165b74312b9cbac845e6f Mon Sep 17 00:00:00 2001 From: Thoren-G Date: Sun, 12 Dec 2021 16:56:30 +0100 Subject: [PATCH 18/27] fixed submodule problem for jsonld/rdfa --- ldcbench.jsonld-gen/rdfa.github.io | 1 - 1 file changed, 1 deletion(-) delete mode 160000 ldcbench.jsonld-gen/rdfa.github.io diff --git a/ldcbench.jsonld-gen/rdfa.github.io b/ldcbench.jsonld-gen/rdfa.github.io deleted file mode 160000 index a730a6e6..00000000 --- a/ldcbench.jsonld-gen/rdfa.github.io +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a730a6e6fafd485d93d506e3182c36fcd18f3a38 From 43809a5115f7d2a27601abd58d89da44ff250113 Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Sun, 12 Dec 2021 21:47:51 +0100 Subject: [PATCH 19/27] Added noOfGraphs as a parameter for integration test --- .../integration-benchmark-parameters-lemming.ttl | 1 + ldcbench.integration-test/integration-benchmark-parameters.ttl | 1 + ldcbench.integration-test/test-benchmark-parameters-lemming.ttl | 1 + ldcbench.integration-test/test-benchmark-parameters.ttl | 1 + 4 files changed, 4 insertions(+) diff --git a/ldcbench.integration-test/integration-benchmark-parameters-lemming.ttl b/ldcbench.integration-test/integration-benchmark-parameters-lemming.ttl index 5cb33c8f..e3165e09 100644 --- a/ldcbench.integration-test/integration-benchmark-parameters-lemming.ttl +++ b/ldcbench.integration-test/integration-benchmark-parameters-lemming.ttl @@ -27,6 +27,7 @@ experiments:New a hobbit:Experiment; :ckanNodeWeight 0; :sparqlNodeWeight 0; :numberOfNodes 4; + :numberOfGraphs 2; :dataset :LemmingTestDataset; :averageNodeGraphDegree 2. diff --git a/ldcbench.integration-test/integration-benchmark-parameters.ttl b/ldcbench.integration-test/integration-benchmark-parameters.ttl index 06707cc4..d4142f28 100644 --- a/ldcbench.integration-test/integration-benchmark-parameters.ttl +++ b/ldcbench.integration-test/integration-benchmark-parameters.ttl @@ -29,4 +29,5 @@ experiments:New a hobbit:Experiment; :microdataNodeWeight 1; :sparqlNodeWeight 0.01; :numberOfNodes 4; + :numberOfGraphs 2; :averageNodeGraphDegree 2. diff --git a/ldcbench.integration-test/test-benchmark-parameters-lemming.ttl b/ldcbench.integration-test/test-benchmark-parameters-lemming.ttl index 732542ea..7d59141e 100644 --- a/ldcbench.integration-test/test-benchmark-parameters-lemming.ttl +++ b/ldcbench.integration-test/test-benchmark-parameters-lemming.ttl @@ -27,6 +27,7 @@ experiments:New a hobbit:Experiment; :ckanNodeWeight 0; :sparqlNodeWeight 0; :numberOfNodes 3; + :numberOfGraphs 2; :dataset :LemmingTestDataset; :averageNodeGraphDegree 2. diff --git a/ldcbench.integration-test/test-benchmark-parameters.ttl b/ldcbench.integration-test/test-benchmark-parameters.ttl index b85ac730..cf3ab96f 100644 --- a/ldcbench.integration-test/test-benchmark-parameters.ttl +++ b/ldcbench.integration-test/test-benchmark-parameters.ttl @@ -36,4 +36,5 @@ experiments:New a hobbit:Experiment; # at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:583) :sparqlNodeWeight 0; :numberOfNodes 3; + :numberOfGraphs 2; :averageNodeGraphDegree 2. From c85bb4964a2b6c28f494c7aaec5f512f67c54a9f Mon Sep 17 00:00:00 2001 From: Thoren-G Date: Sun, 19 Dec 2021 15:42:25 +0100 Subject: [PATCH 20/27] finished work on multiple dumpfile feature --- .../ldcbench/graph/GraphBuilder.java | 16 +++++++++------- .../ldcbench/graph/GrphBasedGraph.java | 12 ++++++++---- .../ldcbench/benchmark/DataGenerator.java | 4 +++- .../ldcbench/benchmark/EvalModule.java | 10 +++++++--- .../ldcbench/nodes/rabbit/GraphHandler.java | 11 +++-------- 5 files changed, 30 insertions(+), 23 deletions(-) diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphBuilder.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphBuilder.java index aefe0131..5f2b09f6 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphBuilder.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GraphBuilder.java @@ -3,7 +3,7 @@ /** * An extension of the {@link Graph} interface offering methods for the addition * of nodes and edges. - * + * * @author Michael Röder (michael.roeder@uni-paderborn.de) * */ @@ -13,7 +13,7 @@ public interface GraphBuilder extends Graph { * Returns a {@link Graph} instance based on the graph data that the builder * collected until now. This instance might offer a faster implementation than * the builder class. - * + * * @return a {@link Graph} instance containing the built graph. */ public default Graph build() { @@ -23,7 +23,7 @@ public default Graph build() { /** * Adds a new edge with the given information if possible. Returns {@code true} * if the edge could be added. - * + * * @param sourceId * the id of the source node * @param targetId @@ -37,7 +37,7 @@ public default Graph build() { /** * Adds a new node to the graph and returns its ID. - * + * * @return the ID of the newly created node */ public int addNode(); @@ -45,7 +45,7 @@ public default Graph build() { /** * Adds the given number of nodes to the graph and returns the range of node * IDs. - * + * * @param nodeCount * the number of nodes that should be added * @return the range of IDs from the first added node ([0], inclusive) to the @@ -56,7 +56,7 @@ public default Graph build() { /** * Sets the IDs of the entrance nodes (i.e., nodes from which all other nodes * can be reached). - * + * * @param entranceNodes * the IDs of the entrance nodes */ @@ -68,7 +68,7 @@ public default Graph build() { * graph. However, to mark external nodes (i.e., nodes that are not part of this * graph but are linked from one of the nodes of this graph), this method can be * used to store their graph ID as well as their ID in the other graph. - * + * * @param nodeId * the ID of the node in this graph * @param graphId @@ -80,4 +80,6 @@ public default Graph build() { * graphID != {@link #INTERNAL_NODE_GRAPH_ID}) */ public void setGraphIdOfNode(int nodeId, int graphId, int externalId); + + public void setGraphId(int i); } diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java index ee515ee5..41e2a36e 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java @@ -34,7 +34,7 @@ public class GrphBasedGraph implements GraphBuilder { * external nodes. */ protected Map externalNodes = new HashMap<>(); - + protected int actualGraphId; //from where can we set this? /** @@ -110,7 +110,7 @@ public int[] outgoingEdgeTargets(int nodeId) { public int[] outgoingEdgeTypes(int nodeId) { return orderedOutgoingEdges(nodeId).map(this::getEdgeType).toArray(); } - + @Override public int outgoingEdgeCount(int nodeId) { return graph.getOutEdgeDegree(nodeId); @@ -123,7 +123,7 @@ public int[] incomingEdgeSources(int nodeId) { public int[] incomingEdgeTypes(int nodeId) { return orderedIncomingEdges(nodeId).map(this::getEdgeType).toArray(); } - + @Override public int incomingEdgeCount(int nodeId) { return graph.getInEdgeDegree(nodeId); @@ -255,8 +255,12 @@ public void setGraphIdOfNode(int nodeId, int graphId, int externalId) { public String toString() { return graph.toString(); } - + public int getGraphId() { return actualGraphId; } + + public void setGraphId(int id) { + actualGraphId = id; + } } diff --git a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java index 58bf827c..bfc7050d 100644 --- a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java +++ b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java @@ -208,7 +208,7 @@ protected void addInterlinks(GraphBuilder g) { Random random = new Random(seedGenerator.getNextSeed()); for (Map.Entry entry : rdfMetadata.entrySet()) { int targetNodeGraph = entry.getKey(); -// GraphMetadata gm = entry.getValue(); + GraphMetadata gm = entry.getValue(); // use random node int nodeWithOutgoingLink = random.nextInt(numberOfInternalNodes); @@ -221,6 +221,7 @@ protected void addInterlinks(GraphBuilder g) { // add a new node int externalNode = g.addNode(); g.setGraphIdOfNode(externalNode, targetNodeGraph, entranceInTargetGraph); + g.setGraphId(gm.graphId); // FIXME don't always use edge type 0 int propertyId = 0; @@ -354,6 +355,7 @@ public void init() throws Exception { GraphBuilder graphs[] = new GraphBuilder[numberOfGraphs]; for (int i = 0; i < numberOfGraphs; i++) { graphs[i] = new GrphBasedGraph(); + graphs[i].setGraphId(i); } if (type == Types.NODE_GRAPH_GENERATOR) { diff --git a/ldcbench.eval-module/src/main/java/org/dice_research/ldcbench/benchmark/EvalModule.java b/ldcbench.eval-module/src/main/java/org/dice_research/ldcbench/benchmark/EvalModule.java index 6a5a1b24..6c89cda0 100644 --- a/ldcbench.eval-module/src/main/java/org/dice_research/ldcbench/benchmark/EvalModule.java +++ b/ldcbench.eval-module/src/main/java/org/dice_research/ldcbench/benchmark/EvalModule.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Date; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Timer; @@ -65,7 +66,7 @@ public class EvalModule extends AbstractPlatformConnectorComponent { protected Semaphore crawlingFinished = new Semaphore(0); protected Semaphore dataGenerationFinished = new Semaphore(0); - protected String graphFiles[]; + protected List graphFiles; private NodeMetadata[] nodeMetadata; protected long startTimeStamp; protected long endTimeStamp; @@ -77,6 +78,8 @@ public class EvalModule extends AbstractPlatformConnectorComponent { public void init() throws Exception { super.init(); + graphFiles = new LinkedList(); + sdk = EnvVariables.getBoolean(ApiConstants.ENV_SDK_KEY, false, LOGGER); // Get the experiment URI @@ -104,7 +107,8 @@ public void handle(NodeMetadata[] body) { public void handle(String[] files) { Arrays.sort(files); LOGGER.debug("Got files: {}", Arrays.toString(files)); - graphFiles = files; + for (String f : files) + graphFiles.add(f); } }; @@ -232,7 +236,7 @@ private void sendResultModel(Model model) throws IOException { private Map runEvaluation() { // Evaluate the results based on the data from the SPARQL storage - TripleBlockStreamSupplier supplier = new FileBasedTripleBlockStreamSupplier(graphFiles, + TripleBlockStreamSupplier supplier = new FileBasedTripleBlockStreamSupplier(graphFiles.toArray(new String[0]), Stream.of(nodeMetadata).map(nm -> nm.getResourceUriTemplate()).toArray(String[]::new), Stream.of(nodeMetadata).map(nm -> nm.getAccessUriTemplate()).toArray(String[]::new), new GraphBasedTripleBlockStreamCreator(), new TTLTarGZBasedTripleBlockStreamCreator()); diff --git a/ldcbench.nodes/src/main/java/org/dice_research/ldcbench/nodes/rabbit/GraphHandler.java b/ldcbench.nodes/src/main/java/org/dice_research/ldcbench/nodes/rabbit/GraphHandler.java index 6b7ae2fd..5a298509 100644 --- a/ldcbench.nodes/src/main/java/org/dice_research/ldcbench/nodes/rabbit/GraphHandler.java +++ b/ldcbench.nodes/src/main/java/org/dice_research/ldcbench/nodes/rabbit/GraphHandler.java @@ -26,14 +26,9 @@ public void run() { try { super.run(); if(!encounteredError()) { - // FIXME fix usage of file [0] - if(receivedFiles.length==1)//if only one file.. then old implementation - handleData(Files.readAllBytes(new File(receivedFiles[0]).toPath())); - else { //handle multiple files. - for (String myfiles : receivedFiles) { - handleData(Files.readAllBytes(new File(myfiles).toPath())); - } - } + for (String myFiles : receivedFiles) { + handleData(Files.readAllBytes(new File(myFiles).toPath())); + } } } catch (Exception e) { LOGGER.error("Error while reading graph. Increasing error count.", e); From b63e9bceeb412cc2def1bdb0b008edf8f600fbc0 Mon Sep 17 00:00:00 2001 From: Thoren-G Date: Sun, 16 Jan 2022 17:49:57 +0100 Subject: [PATCH 21/27] code cleanup for multiple dumpfile feature --- .../ldcbench/graph/GrphBasedGraph.java | 2 +- .../ldcbench/benchmark/DataGenerator.java | 42 +++---------------- .../http/simple/dump/DumpFileResource.java | 1 - 3 files changed, 6 insertions(+), 39 deletions(-) diff --git a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java index 41e2a36e..63172f60 100644 --- a/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java +++ b/ldcbench.api/src/main/java/org/dice_research/ldcbench/graph/GrphBasedGraph.java @@ -35,7 +35,7 @@ public class GrphBasedGraph implements GraphBuilder { */ protected Map externalNodes = new HashMap<>(); - protected int actualGraphId; //from where can we set this? + protected int actualGraphId; /** * Constructor for an empty GraphBuilder. diff --git a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java index bfc7050d..03e13863 100644 --- a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java +++ b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java @@ -231,27 +231,6 @@ protected void addInterlinks(GraphBuilder g) { } } - //TODO remove - // protected void sendFinalGraphs(Graph[] g) throws Exception { - // for(Graph myg : g) { //should i overide serializer as well? - // byte[] data = SerializationHelper.serialize(SERIALIZER_CLASS, myg); - // String name = String.format("graph-%0" + (int) Math.ceil(Math.log10(getNumberOfGenerators() + 1)) + "d" - // + ApiConstants.FILE_ENDING_GRAPH, getNodeId()); - - // // TODO: Use RabbitMQ exchange to send the data (SimpleFileSender doesn't - // // support that) - // try (InputStream is = new ByteArrayInputStream(data); - // SimpleFileSender dataSender = SimpleFileSender.create(outgoingDataQueuefactory, dataQueueName);) { - // dataSender.streamData(is, name); - // } - - // try (InputStream is = new ByteArrayInputStream(data); - // SimpleFileSender dataSender = SimpleFileSender.create(outgoingDataQueuefactory, evalDataQueueName);) { - // dataSender.streamData(is, name); - // } - // } - // } - protected void sendFinalGraph(Graph g) throws Exception { byte[] data = SerializationHelper.serialize(SERIALIZER_CLASS, g); String name = String.format("graph-%0" + (int) Math.ceil(Math.log10(getNumberOfGenerators() + 1)) + "d" @@ -349,9 +328,7 @@ public void init() throws Exception { generator = createRDFGraphGenerator(); } - // GraphBuilder graph = new GrphBasedGraph(); TODO remove - - //initialize two graphs + //initialize all graphs GraphBuilder graphs[] = new GraphBuilder[numberOfGraphs]; for (int i = 0; i < numberOfGraphs; i++) { graphs[i] = new GrphBasedGraph(); @@ -363,10 +340,11 @@ public void init() throws Exception { } if (numberOfNodes != 0) { - LOGGER.debug("Generator {} : Generating {} Graphs", generatorId, numberOfGraphs); + LOGGER.debug("Generator {} : Generating {} graphs", generatorId, numberOfGraphs); for (GraphBuilder mygraph : graphs) { LOGGER.debug("Generator {} : Generating a graph with {} nodes {} average degree and {} seed", generatorId, numberOfNodes, avgDegree, seed); + seed = seedGenerator.getNextSeed(); generator.generateGraph(numberOfNodes/numberOfGraphs, avgDegree, seed, mygraph); } } else { @@ -374,6 +352,7 @@ public void init() throws Exception { generatorId, avgDegree, numberOfEdges, seed); for (GraphBuilder mygraph : graphs) { generator.generateGraph(avgDegree, numberOfEdges, seed, mygraph); + seed = seedGenerator.getNextSeed(); } } @@ -398,18 +377,7 @@ public void init() throws Exception { buf.put(data); dataGeneratorsChannel.basicPublish(dataGeneratorsExchange, "", null, buf.array()); } else { - // Broadcast our graph's metadata. -// GraphMetadata gm = new GraphMetadata(); -// gm.numberOfNodes = graph.getNumberOfNodes(); -// gm.entranceNodes = graph.getEntranceNodes(); -// -// ByteArrayOutputStream buf = new ByteArrayOutputStream(); -// buf.write(header.array(), 0, header.capacity()); -// ObjectOutputStream output = new ObjectOutputStream(buf); -// output.writeObject(gm); -// -// dataGeneratorsChannel.basicPublish(dataGeneratorsExchange, "", null, buf.toByteArray()); - + // Broadcast the graph's metadata for every graph. for (GraphBuilder mygraph : graphs) { GraphMetadata gm = new GraphMetadata(); gm.numberOfNodes = mygraph.getNumberOfNodes(); diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java index 04e610c5..0742311e 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java @@ -56,7 +56,6 @@ public static DumpFileResource create(int domainId, String[] resourceUriTemplate while(it.hasNext()) { archiver.addFileToArchive(aos, it.next()); } - //archiver.buildArchive(archive,dumpFile); contentType = archiver.getMediaType(); return new DumpFileResource(predicate, contentType, archive); } From d19b6e0629dc83594b1dc408357a85d31f4cdbbd Mon Sep 17 00:00:00 2001 From: Thoren-G Date: Sun, 16 Jan 2022 20:08:32 +0100 Subject: [PATCH 22/27] multi_dumpfile small fixes --- .../ldcbench/benchmark/DataGenerator.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java index 5fe3a075..92923282 100644 --- a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java +++ b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java @@ -368,17 +368,17 @@ public void init() throws Exception { if (numberOfNodes != 0) { LOGGER.debug("Generator {} : Generating {} graphs", generatorId, numberOfGraphs); - for (GraphBuilder mygraph : graphs) { + for (int i = 0; i < graphs.length; i++) { LOGGER.debug("Generator {} : Generating a graph with {} nodes {} average degree and {} seed", generatorId, numberOfNodes, avgDegree, seed); seed = seedGenerator.getNextSeed(); - generator.generateGraph(numberOfNodes/numberOfGraphs, avgDegree, seed, mygraph); + generator.generateGraph(numberOfNodes/numberOfGraphs, avgDegree, seed, graphs[i]); } } else { LOGGER.debug("Generator {} : Generating a graph with {} average degree and {} edges and {} seed", generatorId, avgDegree, numberOfEdges, seed); - for (GraphBuilder mygraph : graphs) { - generator.generateGraph(avgDegree, numberOfEdges, seed, mygraph); + for (int i = 0; i < graphs.length; i++) { + generator.generateGraph(avgDegree, numberOfEdges, seed, graphs[i]); seed = seedGenerator.getNextSeed(); } } @@ -386,8 +386,10 @@ public void init() throws Exception { if (type == Types.RDF_GRAPH_GENERATOR) { seed = seedGenerator.getNextSeed(); blankNodesRatio = Double.parseDouble(EnvVariables.getString(ENV_BLANK_NODES_RATIO)); - addBlankNodes(graph, (int) Math.ceil(graph.getNumberOfNodes() * blankNodesRatio), - seed); + for (int i = 0; i < graphs.length; i++) { + addBlankNodes(graphs[i], (int) Math.ceil(graphs[i].getNumberOfNodes() * blankNodesRatio), + seed); + } } if (type == Types.NODE_GRAPH_GENERATOR) { @@ -439,8 +441,8 @@ public void init() throws Exception { LOGGER.info("Got all relevant rdf graphs.", generatorId); - for (GraphBuilder mygraph : graphs) { - addInterlinks(mygraph); + for (int i = 0; i < numberOfGraphs; i++) { + addInterlinks(graphs[i]); } // Send the final graph(s) data. From dfba48c8d652d9f2bff39a2901427f48bad6e8d0 Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Sun, 16 Jan 2022 20:34:32 +0100 Subject: [PATCH 23/27] Removed unecessary test --- .../http/simple/dump/DumpFileBuilder.java | 20 +++++++++---------- .../http/simple/dump/DumpFileBuilderTest.java | 10 +--------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java index 46946f18..70b20244 100755 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java @@ -104,10 +104,11 @@ private OutputStream generateOutputStream() // fileNameBuilder.append('.'); // fileNameBuilder.append(fileExt.get(0)); // } - if(multipleDump==true) { - dumpFile = File.createTempFile("ldcbench", ".dump",path.toFile()); - }else - dumpFile = File.createTempFile("ldcbench", ".dump"); + if(multipleDump) { + dumpFile = File.createTempFile("ldcbench", ".dump",path.toFile()); + } else { + dumpFile = File.createTempFile("ldcbench", ".dump"); + } OutputStream out = new FileOutputStream(dumpFile); out = new BufferedOutputStream(out); if (compression != null) { @@ -206,12 +207,11 @@ public String buildContentType() { } private void generateFolder() { - try { - path = Files.createTempDirectory("multiDump-"); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + try { + path = Files.createTempDirectory("multiDump-"); + } catch (IOException e) { + e.printStackTrace(); + } } } diff --git a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java index 7d353616..5982a6ea 100644 --- a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java +++ b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java @@ -4,8 +4,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -49,7 +47,6 @@ public class DumpFileBuilderTest { private Lang lang; private CompressionStreamFactory compression; private Model expectedModel; - private boolean multipleDump; public DumpFileBuilderTest(int domainId, Graph[] graphs, Lang lang, CompressionStreamFactory compression, Model expectedModel) { @@ -63,17 +60,12 @@ public DumpFileBuilderTest(int domainId, Graph[] graphs, Lang lang, CompressionS @Test public void test() throws NoSuchMethodException, SecurityException, IOException, ReflectiveOperationException, ParserException, NotFoundException { DumpFileBuilder builder = new DumpFileBuilder(domainId, RESOURCE_URI_TEMPLATES, ACCESS_URI_TEMPLATES, graphs, - lang, compression,true); + lang, compression,false); System.out.println( "Testing " + lang + (compression == null ? "" : (" with compression " + compression.getMediaType()))); File file = builder.build(); - if(builder.multipleDump) { - File file2 = builder.build(); - File file3 = builder.build(); - } - Model writtenModel = ModelFactory.createDefaultModel(); if(!lang.equals(LangUtils.HDT_LANG)) { From 8f27203726f01d4e3cbb2932b0e45a670369c74e Mon Sep 17 00:00:00 2001 From: mohandas-pai Date: Sun, 23 Jan 2022 15:10:13 +0100 Subject: [PATCH 24/27] Reverted back the DumpFile Builder class --- .../ldcbench/benchmark/DataGenerator.java | 6 +++--- .../http/simple/SimpleHttpServerComponent.java | 9 +++------ .../nodes/http/simple/dump/DumpFileBuilder.java | 10 ++-------- .../nodes/http/simple/dump/DumpFileResource.java | 15 ++++----------- .../http/simple/dump/DumpFileBuilderTest.java | 2 +- 5 files changed, 13 insertions(+), 29 deletions(-) diff --git a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java index 92923282..7bf9bcde 100644 --- a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java +++ b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java @@ -358,7 +358,7 @@ public void init() throws Exception { //initialize all graphs GraphBuilder graphs[] = new GraphBuilder[numberOfGraphs]; for (int i = 0; i < numberOfGraphs; i++) { - graphs[i] = new GrphBasedGraph(); + graphs[i] = new GrphBasedGraph(); graphs[i].setGraphId(i); } @@ -414,8 +414,8 @@ public void init() throws Exception { dataGeneratorsChannel.basicPublish(dataGeneratorsExchange, "", null, buf.array()); } else { // Broadcast the graph's metadata for every graph. - for (GraphBuilder mygraph : graphs) { - GraphMetadata gm = new GraphMetadata(); + for (GraphBuilder mygraph : graphs) { + GraphMetadata gm = new GraphMetadata(); gm.numberOfNodes = mygraph.getNumberOfNodes(); gm.entranceNodes = mygraph.getEntranceNodes(); gm.graphId = mygraph.getGraphId(); diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java index 5c834ce4..9de0bb62 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java @@ -68,7 +68,6 @@ public class SimpleHttpServerComponent extends NodeComponent implements Componen protected CompressionStreamFactory dumpFileCompression = null; protected Archiver dumpfileArchiver = null; protected int noOfGraphs; - protected boolean multipleFiles=false; @Override public void initBeforeDataGeneration() throws Exception { @@ -76,9 +75,7 @@ public void initBeforeDataGeneration() throws Exception { compressedRatio = Double.parseDouble(EnvVariables.getString(ApiConstants.ENV_COMPRESSED_RATIO_KEY, LOGGER)); disallowedRatio = Double.parseDouble(EnvVariables.getString(ApiConstants.ENV_DISALLOWED_RATIO_KEY, LOGGER)); crawlDelay = EnvVariables.getInt(ApiConstants.ENV_CRAWL_DELAY_KEY, LOGGER); - noOfGraphs = EnvVariables.getInt(ApiConstants.ENV_NUMBER_OF_GRAPHS_KEY,1,LOGGER); - if(noOfGraphs >1) - multipleFiles = true; + noOfGraphs = EnvVariables.getInt(ApiConstants.ENV_NUMBER_OF_GRAPHS_KEY,1,LOGGER); String hostname = InetAddress.getLocalHost().getHostName(); LOGGER.info("Hostname: {}", hostname); @@ -183,9 +180,9 @@ protected Container createContainer() throws Exception { resource = DumpFileResource.create(cloudNodeId.get(), Stream.of(nodeMetadata).map(nm -> nm.getResourceUriTemplate()).toArray(String[]::new), Stream.of(nodeMetadata).map(nm -> nm.getAccessUriTemplate()).toArray(String[]::new), - graphs.toArray(new Graph[graphs.size()]), + graphsArray, r -> r.getPath().toString().equals(dumpFilePath), - dumpFileLang, dumpFileCompression,multipleFiles,dumpfileArchiver); + dumpFileLang, dumpFileCompression,dumpfileArchiver); resources.add(resource); } diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java index 70b20244..3f66ae30 100755 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java @@ -65,18 +65,16 @@ public class DumpFileBuilder { protected final Lang lang; protected final CompressionStreamFactory compression; protected File dumpFile; - protected final boolean multipleDump; protected Path path; public DumpFileBuilder(int domainId, String[] resourceUriTemplates, String[] accessUriTemplates, Graph[] graphs, - Lang lang, CompressionStreamFactory compression,boolean multipleDump) { + Lang lang, CompressionStreamFactory compression) { this.domainId = domainId; this.resourceUriTemplates = resourceUriTemplates; this.accessUriTemplates = accessUriTemplates; this.graphs = graphs; this.lang = lang; this.compression = compression; - this.multipleDump = multipleDump; this.generateFolder(); } @@ -104,11 +102,7 @@ private OutputStream generateOutputStream() // fileNameBuilder.append('.'); // fileNameBuilder.append(fileExt.get(0)); // } - if(multipleDump) { - dumpFile = File.createTempFile("ldcbench", ".dump",path.toFile()); - } else { - dumpFile = File.createTempFile("ldcbench", ".dump"); - } + dumpFile = File.createTempFile("ldcbench", ".dump"); OutputStream out = new FileOutputStream(dumpFile); out = new BufferedOutputStream(out); if (compression != null) { diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java index 4d2fae59..7efd2ced 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java @@ -36,25 +36,18 @@ public class DumpFileResource extends AbstractCrawleableResource { public static final List ARCHIVERS = Arrays.asList(new TarArchiver(), new TarArchiver(ReflectionBasedStreamFactory.create("java.util.zip.GZIPOutputStream", "application/gzip", ".gz")), new ZipArchiver()); - public static final ArrayList multipleFile = new ArrayList(); public static DumpFileResource create(int domainId, String[] resourceUriTemplates, String[] accessUriTemplates, - Graph[] graphs, Predicate predicate, Lang lang, CompressionStreamFactory compression, boolean multipleDump, Archiver archiver) { + Graph[] graphs, Predicate predicate, Lang lang, CompressionStreamFactory compression, Archiver archiver) { DumpFileBuilder builder = new DumpFileBuilder(domainId, resourceUriTemplates, accessUriTemplates, graphs, - lang, compression,multipleDump); + lang, compression); try { File dumpFile = builder.build(); String contentType = builder.buildContentType(); - multipleFile.add(dumpFile); if (archiver != null) { - //Add dump Files to a List and put them into Archive - Iterator it = multipleFile.iterator(); + //Add dump Files to a List and put them into Archive File archive = File.createTempFile("ldcbench", ".archive"); - ArchiveOutputStream aos = archiver.createStream(archive); - while(it.hasNext()) { - archiver.addFileToArchive(aos, it.next()); - } - contentType = archiver.getMediaType(); + archiver.buildArchive(archive, dumpFile); return new DumpFileResource(predicate, contentType, archive); } return new DumpFileResource(predicate, contentType, dumpFile); diff --git a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java index 5982a6ea..4122a8b5 100644 --- a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java +++ b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java @@ -60,7 +60,7 @@ public DumpFileBuilderTest(int domainId, Graph[] graphs, Lang lang, CompressionS @Test public void test() throws NoSuchMethodException, SecurityException, IOException, ReflectiveOperationException, ParserException, NotFoundException { DumpFileBuilder builder = new DumpFileBuilder(domainId, RESOURCE_URI_TEMPLATES, ACCESS_URI_TEMPLATES, graphs, - lang, compression,false); + lang, compression); System.out.println( "Testing " + lang + (compression == null ? "" : (" with compression " + compression.getMediaType()))); From 5e0888c19b79e0bdaaa1326d8fb05ff4d6feaf21 Mon Sep 17 00:00:00 2001 From: Thoren-G Date: Thu, 27 Jan 2022 12:53:31 +0100 Subject: [PATCH 25/27] multiple dumpfile building fixes --- .../simple/SimpleHttpServerComponent.java | 13 ++++++------ .../http/simple/dump/DumpFileBuilder.java | 21 +++++-------------- .../http/simple/dump/DumpFileResource.java | 4 ++-- .../http/simple/dump/DumpFileBuilderTest.java | 7 ++++--- 4 files changed, 18 insertions(+), 27 deletions(-) diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java index 9de0bb62..9e71c513 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/SimpleHttpServerComponent.java @@ -75,7 +75,7 @@ public void initBeforeDataGeneration() throws Exception { compressedRatio = Double.parseDouble(EnvVariables.getString(ApiConstants.ENV_COMPRESSED_RATIO_KEY, LOGGER)); disallowedRatio = Double.parseDouble(EnvVariables.getString(ApiConstants.ENV_DISALLOWED_RATIO_KEY, LOGGER)); crawlDelay = EnvVariables.getInt(ApiConstants.ENV_CRAWL_DELAY_KEY, LOGGER); - noOfGraphs = EnvVariables.getInt(ApiConstants.ENV_NUMBER_OF_GRAPHS_KEY,1,LOGGER); + noOfGraphs = EnvVariables.getInt(ApiConstants.ENV_NUMBER_OF_GRAPHS_KEY,1,LOGGER); String hostname = InetAddress.getLocalHost().getHostName(); LOGGER.info("Hostname: {}", hostname); @@ -178,12 +178,13 @@ protected Container createContainer() throws Exception { if (dumpFileNode) { for(int i=0; i nm.getResourceUriTemplate()).toArray(String[]::new), - Stream.of(nodeMetadata).map(nm -> nm.getAccessUriTemplate()).toArray(String[]::new), - graphsArray, + resourceUriTemplates, + accessUriTemplates, + new Graph[] {graphsArray[i]}, + i, r -> r.getPath().toString().equals(dumpFilePath), - dumpFileLang, dumpFileCompression,dumpfileArchiver); - + dumpFileLang, dumpFileCompression, dumpfileArchiver); + resources.add(resource); } } else { diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java index 3f66ae30..ea5373c0 100755 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilder.java @@ -7,8 +7,6 @@ import java.io.IOException; import java.io.OutputStream; import java.lang.reflect.InvocationTargetException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Arrays; import java.util.List; @@ -65,18 +63,17 @@ public class DumpFileBuilder { protected final Lang lang; protected final CompressionStreamFactory compression; protected File dumpFile; - protected Path path; + protected int dumpfileCount; public DumpFileBuilder(int domainId, String[] resourceUriTemplates, String[] accessUriTemplates, Graph[] graphs, - Lang lang, CompressionStreamFactory compression) { + Lang lang, CompressionStreamFactory compression, int dumpfileCount) { this.domainId = domainId; this.resourceUriTemplates = resourceUriTemplates; this.accessUriTemplates = accessUriTemplates; this.graphs = graphs; this.lang = lang; this.compression = compression; - this.generateFolder(); - + this.dumpfileCount = dumpfileCount; } public File build() @@ -102,7 +99,7 @@ private OutputStream generateOutputStream() // fileNameBuilder.append('.'); // fileNameBuilder.append(fileExt.get(0)); // } - dumpFile = File.createTempFile("ldcbench", ".dump"); + dumpFile = File.createTempFile("ldcbench_" + this.dumpfileCount, ".dump"); OutputStream out = new FileOutputStream(dumpFile); out = new BufferedOutputStream(out); if (compression != null) { @@ -172,7 +169,7 @@ private void writeData(OutputStream out, Lang lang) { TripleIterator iterator; LOGGER.info("Domain ID: " + domainId); LOGGER.info("graph size: " + graphs.length); - + Model model = ModelFactory.createDefaultModel(); org.apache.jena.graph.Graph modelGraph = model.getGraph(); int datasetId = 0; @@ -198,14 +195,6 @@ public String buildContentType() { return compression.getMediaType(); } return lang.getHeaderString(); - } - - private void generateFolder() { - try { - path = Files.createTempDirectory("multiDump-"); - } catch (IOException e) { - e.printStackTrace(); - } } } diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java index 7efd2ced..6b482009 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java @@ -38,9 +38,9 @@ public class DumpFileResource extends AbstractCrawleableResource { new ZipArchiver()); public static DumpFileResource create(int domainId, String[] resourceUriTemplates, String[] accessUriTemplates, - Graph[] graphs, Predicate predicate, Lang lang, CompressionStreamFactory compression, Archiver archiver) { + Graph[] graphs, int dumpfileCount, Predicate predicate, Lang lang, CompressionStreamFactory compression, Archiver archiver) { DumpFileBuilder builder = new DumpFileBuilder(domainId, resourceUriTemplates, accessUriTemplates, graphs, - lang, compression); + lang, compression, dumpfileCount); try { File dumpFile = builder.build(); String contentType = builder.buildContentType(); diff --git a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java index 4122a8b5..9bd22d63 100644 --- a/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java +++ b/ldcbench.http-node/src/test/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileBuilderTest.java @@ -59,15 +59,16 @@ public DumpFileBuilderTest(int domainId, Graph[] graphs, Lang lang, CompressionS @Test public void test() throws NoSuchMethodException, SecurityException, IOException, ReflectiveOperationException, ParserException, NotFoundException { + int dumpfileCount = 0; //only create a single file DumpFileBuilder builder = new DumpFileBuilder(domainId, RESOURCE_URI_TEMPLATES, ACCESS_URI_TEMPLATES, graphs, - lang, compression); + lang, compression, dumpfileCount); System.out.println( "Testing " + lang + (compression == null ? "" : (" with compression " + compression.getMediaType()))); File file = builder.build(); - + Model writtenModel = ModelFactory.createDefaultModel(); - + if(!lang.equals(LangUtils.HDT_LANG)) { try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(file))) { RDFParser parser = RDFParser.create().forceLang(lang).source(in).build(); From a2d89eafaa0f774dc27c866596f731b8920195a6 Mon Sep 17 00:00:00 2001 From: Thoren-G Date: Tue, 1 Feb 2022 15:25:06 +0100 Subject: [PATCH 26/27] code cleanup --- .../org/dice_research/ldcbench/benchmark/DataGenerator.java | 6 +----- .../ldcbench/nodes/http/simple/dump/DumpFileResource.java | 3 --- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java index 0b8cd4d7..585b5475 100644 --- a/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java +++ b/ldcbench.data-generator/src/main/java/org/dice_research/ldcbench/benchmark/DataGenerator.java @@ -116,10 +116,6 @@ public static enum Types { * The number of nodes that should be generated. */ private int numberOfNodes; - /** - * The number of graphs for a node - */ - private int numberOfGraphs; /** * The average degree of the nodes. */ @@ -343,7 +339,7 @@ public void init() throws Exception { seed = seedGenerator.getNextSeed(); numberOfNodes = EnvVariables.getInt(ENV_NUMBER_OF_NODES_KEY, 0); - numberOfGraphs = EnvVariables.getInt(ENV_NUMBER_OF_GRAPHS_KEY, 1); + int numberOfGraphs = EnvVariables.getInt(ENV_NUMBER_OF_GRAPHS_KEY, 1); avgDegree = Double.parseDouble(EnvVariables.getString(ENV_AVERAGE_DEGREE_KEY)); numberOfEdges = EnvVariables.getInt(ENV_NUMBER_OF_EDGES_KEY, 0); diff --git a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java index 6b482009..7056a61a 100644 --- a/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java +++ b/ldcbench.http-node/src/main/java/org/dice_research/ldcbench/nodes/http/simple/dump/DumpFileResource.java @@ -5,13 +5,10 @@ import java.io.FileInputStream; import java.io.InputStream; import java.io.OutputStream; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Iterator; import java.util.List; import java.util.function.Predicate; -import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.io.IOUtils; import org.apache.jena.riot.Lang; import org.dice_research.ldcbench.graph.Graph; From dbefb1045ae2b575e3dc56fd899ed89194e9ceb8 Mon Sep 17 00:00:00 2001 From: Thoren-G Date: Tue, 1 Feb 2022 16:48:26 +0100 Subject: [PATCH 27/27] fixed merge problem --- .../main/java/ldcbench/microdata/gen/MicrodataGenerator.java | 5 +++-- .../java/ldcbench/microformat/gen/MicroformatGenerator.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ldcbench.microdata-gen/src/main/java/ldcbench/microdata/gen/MicrodataGenerator.java b/ldcbench.microdata-gen/src/main/java/ldcbench/microdata/gen/MicrodataGenerator.java index 3a87aa5d..18a15166 100644 --- a/ldcbench.microdata-gen/src/main/java/ldcbench/microdata/gen/MicrodataGenerator.java +++ b/ldcbench.microdata-gen/src/main/java/ldcbench/microdata/gen/MicrodataGenerator.java @@ -20,6 +20,7 @@ import org.apache.commons.io.FileUtils; import org.dice_research.ldcbench.ApiConstants; import org.dice_research.ldcbench.benchmark.DataGenerator; +import org.dice_research.ldcbench.rdf.RDFNodeType; import org.dice_research.ldcbench.graph.Graph; import org.dice_research.ldcbench.rdf.SimpleTripleCreator; import org.dice_research.ldcbench.utils.tar.TarFileGenerator; @@ -97,7 +98,7 @@ protected void generateEntranceFile(Graph graph, Map htmlFiles, Fi protected String generateEntranceNodeUri(Graph graph, SimpleTripleCreator creator) { int entranceNode = graph.getEntranceNodes()[0]; return creator - .createNode(entranceNode, graph.getExternalNodeId(entranceNode), graph.getGraphId(entranceNode), false) + .createNode(entranceNode, graph.getExternalNodeId(entranceNode), graph.getGraphId(entranceNode), RDFNodeType.IRI) .getURI(); } @@ -106,7 +107,7 @@ protected Set generateOutgoingLinks(final Graph graph, final SimpleTripl int entranceNodes[] = graph.getEntranceNodes(); for (int i = 0; i < entranceNodes.length; ++i) { outgoingLinks.addAll(Arrays.stream(graph.outgoingEdgeTargets(entranceNodes[i])) - .mapToObj(id -> creator.createNode(id, graph.getExternalNodeId(id), graph.getGraphId(id), false)) + .mapToObj(id -> creator.createNode(id, graph.getExternalNodeId(id), graph.getGraphId(id), graph.getNodeType(id))) .map(n -> n.getURI()).collect(Collectors.toSet())); } return outgoingLinks; diff --git a/ldcbench.microformat-gen/src/main/java/ldcbench/microformat/gen/MicroformatGenerator.java b/ldcbench.microformat-gen/src/main/java/ldcbench/microformat/gen/MicroformatGenerator.java index bc7e9fad..435a60d0 100644 --- a/ldcbench.microformat-gen/src/main/java/ldcbench/microformat/gen/MicroformatGenerator.java +++ b/ldcbench.microformat-gen/src/main/java/ldcbench/microformat/gen/MicroformatGenerator.java @@ -30,6 +30,7 @@ import org.dice_research.ldcbench.benchmark.DataGenerator; import org.dice_research.ldcbench.graph.Graph; import org.dice_research.ldcbench.microformat.gen.MicroformatEntranceFileGenerator; +import org.dice_research.ldcbench.rdf.RDFNodeType; import org.dice_research.ldcbench.rdf.SimpleTripleCreator; import org.dice_research.ldcbench.utils.tar.TarFileGenerator; import org.hobbit.core.rabbit.RabbitQueueFactory; @@ -111,7 +112,7 @@ protected void generateEntranceFile(Graph graph, Map htmlFiles, Fi protected String generateEntranceNodeUri(Graph graph, SimpleTripleCreator creator) { int entranceNode = graph.getEntranceNodes()[0]; return creator - .createNode(entranceNode, graph.getExternalNodeId(entranceNode), graph.getGraphId(entranceNode), false) + .createNode(entranceNode, graph.getExternalNodeId(entranceNode), graph.getGraphId(entranceNode), RDFNodeType.IRI) .getURI(); } @@ -120,7 +121,7 @@ protected Set generateOutgoingLinks(final Graph graph, final SimpleTripl int entranceNodes[] = graph.getEntranceNodes(); for (int i = 0; i < entranceNodes.length; ++i) { outgoingLinks.addAll(Arrays.stream(graph.outgoingEdgeTargets(entranceNodes[i])) - .mapToObj(id -> creator.createNode(id, graph.getExternalNodeId(id), graph.getGraphId(id), false)) + .mapToObj(id -> creator.createNode(id, graph.getExternalNodeId(id), graph.getGraphId(id), graph.getNodeType(id))) .map(n -> n.getURI()).collect(Collectors.toSet())); } return outgoingLinks;