From 57030654010d96225eca715f7af007fbdc4e76b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Fri, 28 Mar 2025 12:59:51 +0100 Subject: [PATCH 01/13] Write hierarchy db instead of export --- CadRevealComposer/SceneCreator.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CadRevealComposer/SceneCreator.cs b/CadRevealComposer/SceneCreator.cs index f206683b8..5437abe76 100644 --- a/CadRevealComposer/SceneCreator.cs +++ b/CadRevealComposer/SceneCreator.cs @@ -9,6 +9,7 @@ using Commons.Utils; using Configuration; using HierarchyComposer.Functions; +using HierarchyComposer.Model; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Operations; @@ -44,13 +45,11 @@ public static void ExportModelMetadata(DirectoryInfo outputDirectory, ModelMetad File.WriteAllText(metadataPath, metadataString); } - public static void ExportHierarchyDatabase(string databasePath, IReadOnlyList allNodes) + public static void WriteToHierarchyDatabase(string databasePath, IReadOnlyList allNodes) { - var nodes = HierarchyComposerConverter.ConvertToHierarchyNodes(allNodes); - ILogger databaseLogger = NullLogger.Instance; var exporter = new DatabaseComposer(databaseLogger); - exporter.ComposeDatabase(nodes.ToList(), Path.GetFullPath(databasePath)); + exporter.ComposeDatabase(allNodes, Path.GetFullPath(databasePath)); } public static void AddPrioritizedSectorsToDatabase( From f936f24cf501726d114e9830ab242de6a0e3a388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Fri, 28 Mar 2025 13:01:44 +0100 Subject: [PATCH 02/13] Convert hierarchy nodes outside of exporting task to lessen memory load. --- CadRevealComposer/CadRevealComposerRunner.cs | 30 +++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/CadRevealComposer/CadRevealComposerRunner.cs b/CadRevealComposer/CadRevealComposerRunner.cs index d2bffd250..e7fccb2ce 100644 --- a/CadRevealComposer/CadRevealComposerRunner.cs +++ b/CadRevealComposer/CadRevealComposerRunner.cs @@ -67,6 +67,8 @@ IReadOnlyList modelFormatProviders filtering ); + GC.Collect(GC.MaxGeneration, GCCollectionMode.Aggressive, blocking: true); + if (generalMetadata != null) { // Log that we added some metadata @@ -105,19 +107,10 @@ IReadOnlyList modelFormatProviders filtering.PrintFilteringStatsToConsole(); - var exportHierarchyDatabaseTask = Task.Run(() => - { - // Exporting hierarchy on side thread to allow it to run in parallel - var hierarchyExportTimer = Stopwatch.StartNew(); - var databasePath = Path.GetFullPath(Path.Join(outputDirectory.FullName, "hierarchy.db")); - SceneCreator.ExportHierarchyDatabase(databasePath, nodesToExport); - Console.WriteLine( - $"Exported hierarchy database to path \"{databasePath}\" in {hierarchyExportTimer.Elapsed}" - ); - }); - geometriesToProcess = Simplify.OptimizeVertexCountInMeshes(geometriesToProcess); + var exportHierarchyDatabaseTask = WriteHierarchyOnSideThread(outputDirectory, nodesToExport); + var geometriesToProcessArray = geometriesToProcess.ToArray(); if (composerParameters.DevPrimitiveCacheFolder != null) { @@ -145,6 +138,21 @@ IReadOnlyList modelFormatProviders Console.WriteLine($"Convert completed in {totalTimeElapsed.Elapsed}"); } + private static Task WriteHierarchyOnSideThread(DirectoryInfo outputDirectory, IReadOnlyList nodes) + { + var hierarchyNodes = HierarchyComposerConverter.ConvertToHierarchyNodes(nodes); + GC.Collect(GC.MaxGeneration, GCCollectionMode.Aggressive, blocking: true); + return Task.Run(() => + { + var hierarchyExportTimer = Stopwatch.StartNew(); + var databasePath = Path.GetFullPath(Path.Join(outputDirectory.FullName, "hierarchy.db")); + SceneCreator.WriteToHierarchyDatabase(databasePath, hierarchyNodes); + Console.WriteLine( + $"Exported hierarchy database to path \"{databasePath}\" in {hierarchyExportTimer.Elapsed}" + ); + }); + } + public record SplitAndExportResults(List TreeIndexToSectorIdDict); public record TreeIndexSectorIdPair(uint TreeIndex, uint SectorId); From 22b4fac8888bd9d4d9810cddd346d4ace55e36cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Fri, 28 Mar 2025 13:48:51 +0100 Subject: [PATCH 03/13] remove unnecessary garbage collection --- CadRevealComposer/CadRevealComposerRunner.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/CadRevealComposer/CadRevealComposerRunner.cs b/CadRevealComposer/CadRevealComposerRunner.cs index e7fccb2ce..86ab13909 100644 --- a/CadRevealComposer/CadRevealComposerRunner.cs +++ b/CadRevealComposer/CadRevealComposerRunner.cs @@ -67,8 +67,6 @@ IReadOnlyList modelFormatProviders filtering ); - GC.Collect(GC.MaxGeneration, GCCollectionMode.Aggressive, blocking: true); - if (generalMetadata != null) { // Log that we added some metadata @@ -141,7 +139,6 @@ IReadOnlyList modelFormatProviders private static Task WriteHierarchyOnSideThread(DirectoryInfo outputDirectory, IReadOnlyList nodes) { var hierarchyNodes = HierarchyComposerConverter.ConvertToHierarchyNodes(nodes); - GC.Collect(GC.MaxGeneration, GCCollectionMode.Aggressive, blocking: true); return Task.Run(() => { var hierarchyExportTimer = Stopwatch.StartNew(); From f0f02f763247b4b2e48331241bb6eeba141b0e3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Wed, 2 Apr 2025 09:24:05 +0200 Subject: [PATCH 04/13] Remove writing hierarchy on side thread. --- CadRevealComposer/CadRevealComposerRunner.cs | 24 +++++++------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/CadRevealComposer/CadRevealComposerRunner.cs b/CadRevealComposer/CadRevealComposerRunner.cs index 86ab13909..9388bf47b 100644 --- a/CadRevealComposer/CadRevealComposerRunner.cs +++ b/CadRevealComposer/CadRevealComposerRunner.cs @@ -7,7 +7,6 @@ using System.IO; using System.Linq; using System.Text.Json; -using System.Threading.Tasks; using Configuration; using Devtools; using IdProviders; @@ -107,7 +106,7 @@ IReadOnlyList modelFormatProviders geometriesToProcess = Simplify.OptimizeVertexCountInMeshes(geometriesToProcess); - var exportHierarchyDatabaseTask = WriteHierarchyOnSideThread(outputDirectory, nodesToExport); + WriteHierarchy(outputDirectory, nodesToExport); var geometriesToProcessArray = geometriesToProcess.ToArray(); if (composerParameters.DevPrimitiveCacheFolder != null) @@ -124,10 +123,6 @@ IReadOnlyList modelFormatProviders composerParameters ); - if (!exportHierarchyDatabaseTask.IsCompleted) - Console.WriteLine("Waiting for hierarchy export to complete..."); - exportHierarchyDatabaseTask.Wait(); - WriteParametersToParamsFile(modelParameters, composerParameters, outputDirectory); ModifyHierarchyPostProcess(outputDirectory, splitExportResults); @@ -136,18 +131,15 @@ IReadOnlyList modelFormatProviders Console.WriteLine($"Convert completed in {totalTimeElapsed.Elapsed}"); } - private static Task WriteHierarchyOnSideThread(DirectoryInfo outputDirectory, IReadOnlyList nodes) + private static void WriteHierarchy(DirectoryInfo outputDirectory, IReadOnlyList nodes) { var hierarchyNodes = HierarchyComposerConverter.ConvertToHierarchyNodes(nodes); - return Task.Run(() => - { - var hierarchyExportTimer = Stopwatch.StartNew(); - var databasePath = Path.GetFullPath(Path.Join(outputDirectory.FullName, "hierarchy.db")); - SceneCreator.WriteToHierarchyDatabase(databasePath, hierarchyNodes); - Console.WriteLine( - $"Exported hierarchy database to path \"{databasePath}\" in {hierarchyExportTimer.Elapsed}" - ); - }); + var hierarchyExportTimer = Stopwatch.StartNew(); + var databasePath = Path.GetFullPath(Path.Join(outputDirectory.FullName, "hierarchy.db")); + SceneCreator.WriteToHierarchyDatabase(databasePath, hierarchyNodes); + Console.WriteLine( + $"Exported hierarchy database to path \"{databasePath}\" in {hierarchyExportTimer.Elapsed}" + ); } public record SplitAndExportResults(List TreeIndexToSectorIdDict); From a41f141a70f4e7ceb51866c8c829d55237a9104a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Wed, 2 Apr 2025 10:50:27 +0200 Subject: [PATCH 05/13] debugging console writes --- CadRevealComposer/CadRevealComposerRunner.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CadRevealComposer/CadRevealComposerRunner.cs b/CadRevealComposer/CadRevealComposerRunner.cs index 9388bf47b..e3fa3bc2f 100644 --- a/CadRevealComposer/CadRevealComposerRunner.cs +++ b/CadRevealComposer/CadRevealComposerRunner.cs @@ -133,9 +133,13 @@ IReadOnlyList modelFormatProviders private static void WriteHierarchy(DirectoryInfo outputDirectory, IReadOnlyList nodes) { + Console.WriteLine("WriteHieararchy line 136"); var hierarchyNodes = HierarchyComposerConverter.ConvertToHierarchyNodes(nodes); + Console.WriteLine("WriteHieararchy line 138"); var hierarchyExportTimer = Stopwatch.StartNew(); + Console.WriteLine("WriteHieararchy line 140"); var databasePath = Path.GetFullPath(Path.Join(outputDirectory.FullName, "hierarchy.db")); + Console.WriteLine("WriteHieararchy line 142"); SceneCreator.WriteToHierarchyDatabase(databasePath, hierarchyNodes); Console.WriteLine( $"Exported hierarchy database to path \"{databasePath}\" in {hierarchyExportTimer.Elapsed}" From 564362ca69d69e68e61a1a8d8d6f6f31f732075e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Thu, 3 Apr 2025 09:38:03 +0200 Subject: [PATCH 06/13] dlete console writelines --- CadRevealComposer/CadRevealComposerRunner.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CadRevealComposer/CadRevealComposerRunner.cs b/CadRevealComposer/CadRevealComposerRunner.cs index e3fa3bc2f..9388bf47b 100644 --- a/CadRevealComposer/CadRevealComposerRunner.cs +++ b/CadRevealComposer/CadRevealComposerRunner.cs @@ -133,13 +133,9 @@ IReadOnlyList modelFormatProviders private static void WriteHierarchy(DirectoryInfo outputDirectory, IReadOnlyList nodes) { - Console.WriteLine("WriteHieararchy line 136"); var hierarchyNodes = HierarchyComposerConverter.ConvertToHierarchyNodes(nodes); - Console.WriteLine("WriteHieararchy line 138"); var hierarchyExportTimer = Stopwatch.StartNew(); - Console.WriteLine("WriteHieararchy line 140"); var databasePath = Path.GetFullPath(Path.Join(outputDirectory.FullName, "hierarchy.db")); - Console.WriteLine("WriteHieararchy line 142"); SceneCreator.WriteToHierarchyDatabase(databasePath, hierarchyNodes); Console.WriteLine( $"Exported hierarchy database to path \"{databasePath}\" in {hierarchyExportTimer.Elapsed}" From dcaed1f7a25cecbce5df8bdfa75e2e6813953ef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Thu, 3 Apr 2025 09:38:35 +0200 Subject: [PATCH 07/13] Check memory usage and write to console --- .../Functions/DatabaseComposer.cs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/HierarchyComposer/Functions/DatabaseComposer.cs b/HierarchyComposer/Functions/DatabaseComposer.cs index e4e79fb1e..3616abb83 100644 --- a/HierarchyComposer/Functions/DatabaseComposer.cs +++ b/HierarchyComposer/Functions/DatabaseComposer.cs @@ -22,8 +22,26 @@ public DatabaseComposer(ILogger? logger = null) } // ReSharper disable once CognitiveComplexity + + // Method to check and write current memory usage to the console + static void CheckMemoryUsage(string currentLine) + { + // Get the current process + Process currentProcess = Process.GetCurrentProcess(); + + // Get the physical memory usage (in bytes) + long totalBytesOfMemoryUsed = currentProcess.WorkingSet64; + + // Convert to megabytes for easier reading + double megabytesUsed = totalBytesOfMemoryUsed / (1024.0 * 1024.0); + + // Write the memory usage to the console + Console.WriteLine($"Memory usage (MB): {megabytesUsed:N2} at line {currentLine}"); + } + public void ComposeDatabase(IReadOnlyList inputNodes, string outputDatabaseFullPath) { + CheckMemoryUsage("44"); if (File.Exists(outputDatabaseFullPath)) File.Delete(outputDatabaseFullPath); @@ -57,6 +75,8 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp _logger.LogInformation("Creating database model entries"); long pdmsEntryIdCounter = 0; + CheckMemoryUsage("78"); + var pdmsEntries = jsonPdmsKeyValuePairs .GroupBy(kvp => kvp.GetGroupKey()) .ToDictionary( @@ -74,6 +94,8 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp .GroupBy(b => b.GetGroupKey()) .ToDictionary(keySelector: g => g.Key, elementSelector: g => g.First().CopyWithNewId(++aabbIdCounter)); + CheckMemoryUsage("97"); + var nodes = inputNodes .Select(inputNode => new Node { @@ -98,6 +120,8 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp }) .ToDictionary(n => n.Id, n => n); + CheckMemoryUsage("123"); + var nodePdmsEntries = nodes.Values.Where(n => n.NodePDMSEntry != null).SelectMany(n => n.NodePDMSEntry!); var sqliteComposeTimer = MopTimer.Create("Populating database and building index", _logger); @@ -105,6 +129,8 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp using var connection = new SqliteConnection(connectionString); connection.Open(); + CheckMemoryUsage("132"); + // ReSharper disable AccessToDisposedClosure MopTimer.RunAndMeasure( "Insert PDMSEntries", @@ -120,6 +146,8 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); + CheckMemoryUsage("149"); + MopTimer.RunAndMeasure( "Insert NodePDMSEntries", _logger, @@ -134,6 +162,8 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); + CheckMemoryUsage("165"); + MopTimer.RunAndMeasure( "Insert AABBs", _logger, @@ -154,6 +184,8 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); + CheckMemoryUsage("187"); + MopTimer.RunAndMeasure( "Insert Nodes", _logger, @@ -167,6 +199,8 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); + CheckMemoryUsage("202"); + MopTimer.RunAndMeasure( "Creating indexes", _logger, @@ -188,6 +222,8 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); + CheckMemoryUsage("225"); + MopTimer.RunAndMeasure( "Optimizing Database", _logger, @@ -204,6 +240,8 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); + CheckMemoryUsage("243"); + MopTimer.RunAndMeasure( "VACUUM Database", _logger, @@ -244,6 +282,8 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); + CheckMemoryUsage("285"); + // ReSharper restore AccessToDisposedClosure sqliteComposeTimer.LogCompletion(); } From c9d24c127c6758958bc4cd6121913799d4f97b54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Thu, 3 Apr 2025 11:09:27 +0200 Subject: [PATCH 08/13] IEnumerable instead of list --- HierarchyComposer/Functions/DatabaseComposer.cs | 13 ++++++------- HierarchyComposer/Model/Node.cs | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/HierarchyComposer/Functions/DatabaseComposer.cs b/HierarchyComposer/Functions/DatabaseComposer.cs index 3616abb83..c4bf90158 100644 --- a/HierarchyComposer/Functions/DatabaseComposer.cs +++ b/HierarchyComposer/Functions/DatabaseComposer.cs @@ -108,13 +108,12 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp HasMesh = inputNode.HasMesh, ParentId = inputNode.ParentId, TopNodeId = inputNode.TopNodeId, - NodePDMSEntry = inputNode - .PDMSData.Select(kvp => new NodePDMSEntry - { - NodeId = inputNode.NodeId, - PDMSEntryId = pdmsEntries[kvp.GetGroupKey()].Id - }) - .ToList(), + NodePDMSEntry = inputNode.PDMSData + .Select(kvp => new NodePDMSEntry + { + NodeId = inputNode.NodeId, + PDMSEntryId = pdmsEntries[kvp.GetGroupKey()].Id + }), AABB = inputNode.AABB == null ? null : aabbs[inputNode.AABB.GetGroupKey()], DiagnosticInfo = inputNode.OptionalDiagnosticInfo }) diff --git a/HierarchyComposer/Model/Node.cs b/HierarchyComposer/Model/Node.cs index 852555c43..3473a8b56 100644 --- a/HierarchyComposer/Model/Node.cs +++ b/HierarchyComposer/Model/Node.cs @@ -27,7 +27,7 @@ public class Node public uint TopNodeId { get; init; } - public virtual ICollection? NodePDMSEntry { get; init; } = null!; + public virtual IEnumerable? NodePDMSEntry { get; init; } = null!; public uint? AABBId { get; init; } From 5d5ae5dc34b102fe2477e3be392454e0c626d578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Thu, 3 Apr 2025 11:09:39 +0200 Subject: [PATCH 09/13] do not cast to array --- HierarchyComposer/Functions/DatabaseComposer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HierarchyComposer/Functions/DatabaseComposer.cs b/HierarchyComposer/Functions/DatabaseComposer.cs index c4bf90158..e52449c53 100644 --- a/HierarchyComposer/Functions/DatabaseComposer.cs +++ b/HierarchyComposer/Functions/DatabaseComposer.cs @@ -68,7 +68,7 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp var jsonPdmsKeyValuePairs = MopTimer.RunAndMeasure( "Collecting PDMS data", _logger, - () => inputNodes.SelectMany(n => n.PDMSData).ToArray() + () => inputNodes.SelectMany(n => n.PDMSData) ); var jsonAabbs = inputNodes.Where(jn => jn.AABB != null).Select(jn => jn.AABB!); From ee048abd5d826b3f0bd321204dc052d3e66795ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Thu, 3 Apr 2025 11:11:47 +0200 Subject: [PATCH 10/13] indent lines --- HierarchyComposer/Functions/DatabaseComposer.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/HierarchyComposer/Functions/DatabaseComposer.cs b/HierarchyComposer/Functions/DatabaseComposer.cs index e52449c53..4df74907b 100644 --- a/HierarchyComposer/Functions/DatabaseComposer.cs +++ b/HierarchyComposer/Functions/DatabaseComposer.cs @@ -109,11 +109,11 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp ParentId = inputNode.ParentId, TopNodeId = inputNode.TopNodeId, NodePDMSEntry = inputNode.PDMSData - .Select(kvp => new NodePDMSEntry - { - NodeId = inputNode.NodeId, - PDMSEntryId = pdmsEntries[kvp.GetGroupKey()].Id - }), + .Select(kvp => new NodePDMSEntry + { + NodeId = inputNode.NodeId, + PDMSEntryId = pdmsEntries[kvp.GetGroupKey()].Id + }), AABB = inputNode.AABB == null ? null : aabbs[inputNode.AABB.GetGroupKey()], DiagnosticInfo = inputNode.OptionalDiagnosticInfo }) From 6e94e16a67292fbc5c6f5fb2067f0e951b44f966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Thu, 3 Apr 2025 11:13:24 +0200 Subject: [PATCH 11/13] formatting --- CadRevealComposer/CadRevealComposerRunner.cs | 4 +--- HierarchyComposer/Functions/DatabaseComposer.cs | 11 +++++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/CadRevealComposer/CadRevealComposerRunner.cs b/CadRevealComposer/CadRevealComposerRunner.cs index 9388bf47b..05484dcd6 100644 --- a/CadRevealComposer/CadRevealComposerRunner.cs +++ b/CadRevealComposer/CadRevealComposerRunner.cs @@ -137,9 +137,7 @@ private static void WriteHierarchy(DirectoryInfo outputDirectory, IReadOnlyList< var hierarchyExportTimer = Stopwatch.StartNew(); var databasePath = Path.GetFullPath(Path.Join(outputDirectory.FullName, "hierarchy.db")); SceneCreator.WriteToHierarchyDatabase(databasePath, hierarchyNodes); - Console.WriteLine( - $"Exported hierarchy database to path \"{databasePath}\" in {hierarchyExportTimer.Elapsed}" - ); + Console.WriteLine($"Exported hierarchy database to path \"{databasePath}\" in {hierarchyExportTimer.Elapsed}"); } public record SplitAndExportResults(List TreeIndexToSectorIdDict); diff --git a/HierarchyComposer/Functions/DatabaseComposer.cs b/HierarchyComposer/Functions/DatabaseComposer.cs index 4df74907b..4a93f61dc 100644 --- a/HierarchyComposer/Functions/DatabaseComposer.cs +++ b/HierarchyComposer/Functions/DatabaseComposer.cs @@ -108,12 +108,11 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp HasMesh = inputNode.HasMesh, ParentId = inputNode.ParentId, TopNodeId = inputNode.TopNodeId, - NodePDMSEntry = inputNode.PDMSData - .Select(kvp => new NodePDMSEntry - { - NodeId = inputNode.NodeId, - PDMSEntryId = pdmsEntries[kvp.GetGroupKey()].Id - }), + NodePDMSEntry = inputNode.PDMSData.Select(kvp => new NodePDMSEntry + { + NodeId = inputNode.NodeId, + PDMSEntryId = pdmsEntries[kvp.GetGroupKey()].Id + }), AABB = inputNode.AABB == null ? null : aabbs[inputNode.AABB.GetGroupKey()], DiagnosticInfo = inputNode.OptionalDiagnosticInfo }) From c3bb9297cfaaa9d84fb4a121d851810d65905b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Thu, 3 Apr 2025 14:58:29 +0200 Subject: [PATCH 12/13] Really hacky batching of node dict creation. --- .../Functions/DatabaseComposer.cs | 101 +++++++++++++----- 1 file changed, 73 insertions(+), 28 deletions(-) diff --git a/HierarchyComposer/Functions/DatabaseComposer.cs b/HierarchyComposer/Functions/DatabaseComposer.cs index 4a93f61dc..cd96d16db 100644 --- a/HierarchyComposer/Functions/DatabaseComposer.cs +++ b/HierarchyComposer/Functions/DatabaseComposer.cs @@ -96,29 +96,74 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp CheckMemoryUsage("97"); - var nodes = inputNodes - .Select(inputNode => new Node + // Process nodes in smaller batches to reduce memory usage + var nodesBatchSize = 1000; // Adjust batch size as needed + var nodes = new Dictionary(); + var nodeBatches = inputNodes.Chunk(nodesBatchSize); + int i = 0; + foreach (var batch in nodeBatches) + { + var batchNodes = batch + .Select(inputNode => new Node + { + Id = inputNode.NodeId, + EndId = inputNode.EndId, + RefNoPrefix = inputNode.RefNoPrefix, + RefNoDb = inputNode.RefNoDb, + RefNoSequence = inputNode.RefNoSequence, + Name = inputNode.Name, + HasMesh = inputNode.HasMesh, + ParentId = inputNode.ParentId, + TopNodeId = inputNode.TopNodeId, + NodePDMSEntry = inputNode.PDMSData.Select(kvp => new NodePDMSEntry + { + NodeId = inputNode.NodeId, + PDMSEntryId = pdmsEntries[kvp.GetGroupKey()].Id + }), + AABB = inputNode.AABB == null ? null : aabbs[inputNode.AABB.GetGroupKey()], + DiagnosticInfo = inputNode.OptionalDiagnosticInfo + }) + .ToDictionary(n => n.Id, n => n); + try { - Id = inputNode.NodeId, - EndId = inputNode.EndId, - RefNoPrefix = inputNode.RefNoPrefix, - RefNoDb = inputNode.RefNoDb, - RefNoSequence = inputNode.RefNoSequence, - Name = inputNode.Name, - HasMesh = inputNode.HasMesh, - ParentId = inputNode.ParentId, - TopNodeId = inputNode.TopNodeId, - NodePDMSEntry = inputNode.PDMSData.Select(kvp => new NodePDMSEntry + foreach (var kvp in batchNodes) { - NodeId = inputNode.NodeId, - PDMSEntryId = pdmsEntries[kvp.GetGroupKey()].Id - }), - AABB = inputNode.AABB == null ? null : aabbs[inputNode.AABB.GetGroupKey()], - DiagnosticInfo = inputNode.OptionalDiagnosticInfo - }) - .ToDictionary(n => n.Id, n => n); + nodes[kvp.Key] = kvp.Value; + } + } + catch (Exception) + { + throw new ArgumentException("nodes key already exists "); // TODO: DELETE THIS, only for testing /kag + } + i += 1; + CheckMemoryUsage($"Chunk {i} processed"); + } + - CheckMemoryUsage("123"); + // + // var nodes = inputNodes + // .Select(inputNode => new Node + // { + // Id = inputNode.NodeId, + // EndId = inputNode.EndId, + // RefNoPrefix = inputNode.RefNoPrefix, + // RefNoDb = inputNode.RefNoDb, + // RefNoSequence = inputNode.RefNoSequence, + // Name = inputNode.Name, + // HasMesh = inputNode.HasMesh, + // ParentId = inputNode.ParentId, + // TopNodeId = inputNode.TopNodeId, + // NodePDMSEntry = inputNode.PDMSData.Select(kvp => new NodePDMSEntry + // { + // NodeId = inputNode.NodeId, + // PDMSEntryId = pdmsEntries[kvp.GetGroupKey()].Id + // }), + // AABB = inputNode.AABB == null ? null : aabbs[inputNode.AABB.GetGroupKey()], + // DiagnosticInfo = inputNode.OptionalDiagnosticInfo + // }) + // .ToDictionary(n => n.Id, n => n); + + CheckMemoryUsage("166"); var nodePdmsEntries = nodes.Values.Where(n => n.NodePDMSEntry != null).SelectMany(n => n.NodePDMSEntry!); @@ -127,7 +172,7 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp using var connection = new SqliteConnection(connectionString); connection.Open(); - CheckMemoryUsage("132"); + CheckMemoryUsage("175"); // ReSharper disable AccessToDisposedClosure MopTimer.RunAndMeasure( @@ -144,7 +189,7 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); - CheckMemoryUsage("149"); + CheckMemoryUsage("192"); MopTimer.RunAndMeasure( "Insert NodePDMSEntries", @@ -160,7 +205,7 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); - CheckMemoryUsage("165"); + CheckMemoryUsage("208"); MopTimer.RunAndMeasure( "Insert AABBs", @@ -182,7 +227,7 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); - CheckMemoryUsage("187"); + CheckMemoryUsage("230"); MopTimer.RunAndMeasure( "Insert Nodes", @@ -197,7 +242,7 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); - CheckMemoryUsage("202"); + CheckMemoryUsage("245"); MopTimer.RunAndMeasure( "Creating indexes", @@ -220,7 +265,7 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); - CheckMemoryUsage("225"); + CheckMemoryUsage("268"); MopTimer.RunAndMeasure( "Optimizing Database", @@ -238,7 +283,7 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); - CheckMemoryUsage("243"); + CheckMemoryUsage("286"); MopTimer.RunAndMeasure( "VACUUM Database", @@ -280,7 +325,7 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp } ); - CheckMemoryUsage("285"); + CheckMemoryUsage("Last line"); // ReSharper restore AccessToDisposedClosure sqliteComposeTimer.LogCompletion(); From cad236f81b1a8b65aeb0d15eac0460ebeb51b813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arvid=20Gr=C3=A4ns?= <79848215+ArGr1@users.noreply.github.com> Date: Thu, 3 Apr 2025 15:00:21 +0200 Subject: [PATCH 13/13] linting --- HierarchyComposer/Functions/DatabaseComposer.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/HierarchyComposer/Functions/DatabaseComposer.cs b/HierarchyComposer/Functions/DatabaseComposer.cs index cd96d16db..6d5fa2309 100644 --- a/HierarchyComposer/Functions/DatabaseComposer.cs +++ b/HierarchyComposer/Functions/DatabaseComposer.cs @@ -139,7 +139,6 @@ public void ComposeDatabase(IReadOnlyList inputNodes, string outp CheckMemoryUsage($"Chunk {i} processed"); } - // // var nodes = inputNodes // .Select(inputNode => new Node