From 50491f4b2fb9cc7a6ea5317bf569d732e1591964 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Tue, 3 Feb 2026 10:37:11 -0500 Subject: [PATCH 1/8] Improve file system repo caching --- CHANGELOG.md | 1 + src/cls/IPM/Main.cls | 23 ++++ src/cls/IPM/Repo/Filesystem/Cache.cls | 74 +++++++++++ src/cls/IPM/Repo/Filesystem/Definition.cls | 121 ++++++++++++++---- .../IPM/Repo/Filesystem/PackageService.cls | 15 ++- 5 files changed, 202 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33ae8581e..6929969ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - #1024: Added flag -export-python-deps to publish command +- #536: When installing from a file system repos, if the cached version of the module is stale (the modification time of the `module.xml` is different than when it was cached), pull again from disk. Also add `-rebuild-cache` flag for the `repo` command has been to rebuild the file system repo's entire cache. ### Fixed - #996: Ensure COS commands execute in exec under a dedicated, isolated context diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index 64f7e2819..16d00a36f 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -2146,6 +2146,22 @@ ClassMethod Repository(ByRef pCommandInfo) [ Internal ] set tType = "%IPM.Repo.Remote.Definition" $$$ThrowOnError($classmethod(tType,"Configure",1,.tModifiers,.tData)) do ..Shell("repo -list") + } elseif $$$HasModifier(pCommandInfo,"rebuild-cache") { + // Validate repository exists and is filesystem type + set repoName = $$$GetModifier(pCommandInfo,"name") + set serverDef = ##class(%IPM.Repo.Definition).ServerDefinitionKeyOpen(repoName,,.sc) + $$$ThrowOnError(sc) + + if 'serverDef.%IsA("%IPM.Repo.Filesystem.Definition") { + $$$ThrowStatus($$$ERROR($$$GeneralError,"Cache rebuild only supported for filesystem repositories")) + } + + // Rebuild cache + write !,"Rebuilding cache for repository: ",repoName + set sc = serverDef.BuildCache(1, 1) // purge=1, verbose=1 + $$$ThrowOnError(sc) + + write !,"Cache rebuilt successfully" } else { set tName = $$$GetModifier(pCommandInfo,"name") set tType = $listget(serverClassList) @@ -2247,6 +2263,13 @@ ClassMethod ShowModulesForRepository( set list("width") = width write ! do ..DisplayModules(.list) + + // If filesystem repo, show last cache rebuild time and command to rebuild it since there's no auto full cache rebuild mechanism (other than reconfiguring the repo entirely) + set server = ##class(%IPM.Repo.Definition).ServerDefinitionKeyOpen(pRepoName,,.tSC) + $$$ThrowOnError(tSC) + if server.%IsA("%IPM.Repo.Filesystem.Definition") { + write !,"Last full cache rebuild for '", pRepoName, "' was on ", server.CacheLastRebuilt, " (UTC). Run 'repo -n "_pRepoName_" -rebuild-cache' to rebuild the cache.",! + } } Query SourceControlClasses() As %SQLQuery(ROWSPEC = "ID:%String,Name:%String") [ SqlProc ] diff --git a/src/cls/IPM/Repo/Filesystem/Cache.cls b/src/cls/IPM/Repo/Filesystem/Cache.cls index 03edd9f2f..68891ce9b 100644 --- a/src/cls/IPM/Repo/Filesystem/Cache.cls +++ b/src/cls/IPM/Repo/Filesystem/Cache.cls @@ -15,6 +15,9 @@ Property SubDirectory As %String(MAXLEN = 260); Property LastModified As %TimeStamp [ Required ]; +/// Filesystem modification time of the module.xml file +Property FileMTime As %TimeStamp; + /// Full module manifest Property Manifest As %Stream.GlobalCharacter; @@ -113,6 +116,74 @@ Method HandleSaveError(pSC As %Status) As %Status quit tSC } +/// Open a cache entry and validate it against the filesystem +/// If the cached entry is stale (FileMTime differs from actual file mtime), +/// re-parse the module.xml and update the cache entry +/// Returns the cache object (possibly refreshed) or throws error if not found +ClassMethod RootNameVersionOpenValidated( + root As %String, + name As %String, + versionString As %String, + concurrency As %Integer = -1, + Output status As %Status) As %IPM.Repo.Filesystem.Cache +{ + set status = $$$OK + set cacheObj = "" + + try { + // Open the cache entry + set cacheObj = ..RootNameVersionOpen(root, name, versionString, concurrency, .status) + $$$ThrowOnError(status) + + set defObj = ##class(%IPM.Repo.Filesystem.Definition).RootIndexOpen(root, , .status) + $$$ThrowOnError(status) + + // Check for schema migration (old cache entries without FileMTime) + set dirPath = ##class(%File).NormalizeFilename(cacheObj.SubDirectory, cacheObj.Root) + set filePath = ##class(%File).NormalizeFilename("module.xml", dirPath) + if (cacheObj.FileMTime = "") { + // Old cache entry - refresh it to populate FileMTime + if ##class(%File).Exists(filePath) { + set horologTime = ##class(%Library.File).GetFileDateModified(filePath) + set currentMTime = $zdatetime(horologTime, 3) + do defObj.RefreshCacheEntry(cacheObj, filePath, currentMTime) + $$$ThrowOnError(status) + } else { + // File doesn't exist - invalidate cache + set status = $$$ERROR($$$GeneralError, "module.xml not found in " _ dirPath) + do cacheObj.%DeleteId(cacheObj.%Id()) + set cacheObj = "" + quit + } + } + + // Stat the file to get current mtime + if '##class(%File).Exists(filePath) { + // File deleted from filesystem - invalidate cache + set status = $$$ERROR($$$GeneralError, "module.xml not found in " _ dirPath) + do cacheObj.%DeleteId(cacheObj.%Id()) + set cacheObj = "" + quit + } + + set horologTime = ##class(%Library.File).GetFileDateModified(filePath) + set currentMTime = $zdatetime(horologTime, 3) // ODBC format + + // Compare cached mtime with current mtime + if (cacheObj.FileMTime '= currentMTime) { + do defObj.RefreshCacheEntry(cacheObj, filePath, currentMTime) + if $$$ISERR(status) { + quit + } + } + } catch ex { + set status = ex.AsStatus() + set cacheObj = "" + } + + quit cacheObj +} + Storage Default { @@ -164,6 +235,9 @@ Storage Default DisplayName + +FileMTime + ^IPM.Repo.Filesystem.CacheD CacheDefaultData diff --git a/src/cls/IPM/Repo/Filesystem/Definition.cls b/src/cls/IPM/Repo/Filesystem/Definition.cls index 582b32405..5e9e4b696 100644 --- a/src/cls/IPM/Repo/Filesystem/Definition.cls +++ b/src/cls/IPM/Repo/Filesystem/Definition.cls @@ -17,6 +17,9 @@ Property Root As %String(MAXLEN = 260) [ Required ]; /// How many levels of depth to search for module.xml files; 0 indicates unlimited. Property Depth As %Integer [ InitialExpression = 0, Required ]; +/// Timestamp of last cache rebuild for this repository +Property CacheLastRebuilt As %TimeStamp; + /// Prompt to use for Root in interactive configuration of this repository type Parameter RootPromptString = {$$$Text("Root File Path:","ZPM")}; @@ -29,6 +32,7 @@ XData Commands + repo -name LocalFiles -snapshots 1 -fs -depth 2 -path C:\MyWorkspace\RootModuleDir\ @@ -114,6 +118,25 @@ Trigger RootChanged [ Event = UPDATE, Foreach = row/object ] } } +/// When Depth property changes, cache needs full rebuild +/// (different depth means different set of files included) +Trigger DepthChanged [ Event = UPDATE, Foreach = row/object ] +{ + new oldDepth, newDepth + if ({Depth*C}) { + set oldDepth = {Depth*O} + set newDepth = {Depth} + + // Depth changed - rebuild cache + set oref = ..%Open({ID}) + if $isobject(oref) { + do oref.BuildCache(1, 0) // Full rebuild, non-verbose + } + } +} + +/// Build the cache of modules found in the filesystem repository. +/// pPurge - if 1, deletes existing cache entries before rebuilding Method BuildCache( pPurge As %Boolean = 1, pVerbose As %Integer = 0, @@ -173,6 +196,10 @@ Method BuildCache( write:pVerbose !,tName," ",tVersionString," @ ",##class(%File).NormalizeDirectory(..Root_tSubDirectory) } $$$ThrowOnError(tAggSC) + + // set cache rebuild timestamp using UTC + set ..CacheLastRebuilt = $zdatetime($ztimestamp,3) + $$$ThrowOnError(..%Save()) tcommit } catch e { set tSC = e.AsStatus() @@ -181,6 +208,61 @@ Method BuildCache( quit tSC } +/// Parse a module.xml file and return parsed metadata +Method ParseModuleFile( + filePath As %String, + root As %String, + Output name As %String, + Output versionString As %String, + Output stream As %Stream.GlobalCharacter) +{ + set name = "" + set versionString = "" + + // Validate file is a valid Studio document export + $$$ThrowOnError($system.OBJ.Load(filePath, "-d", , .loadedList, 1)) + + if ($length(loadedList, ",") > 1) { + $$$ThrowStatus($$$ERROR($$$GeneralError, "File contains multiple documents")) + } + + set ext = $zconvert($piece($get(loadedList), ".", *), "U") + if (ext '= "ZPM") { + $$$ThrowStatus($$$ERROR($$$GeneralError, "File is not a ZPM module")) + } + + // Parse module info + kill stream, name, versionString + $$$ThrowOnError(..GetModuleStreamFromFile(filePath, .stream, .name, .versionString)) +} + +/// Refresh a cache entry by re-parsing its module.xml file +/// Updates the cache entry in-place with new manifest and mtime +Method RefreshCacheEntry( + cacheObj As %IPM.Repo.Filesystem.Cache, + filePath As %String, + newMTime As %TimeStamp) +{ + // Parse the module file + kill stream, name, versionString + do ..ParseModuleFile(filePath, cacheObj.Root, .name, .versionString, .stream) + + // Update the cache entry + set cacheObj.Name = name + set cacheObj.VersionString = versionString + set cacheObj.FileMTime = newMTime + do cacheObj.Manifest.Clear() + do cacheObj.Manifest.CopyFrom(stream) + set cacheObj.LastModified = $zdatetime($ztimestamp, 3) + + set saveSC = cacheObj.%Save() + if $$$ISERR(saveSC) { + set status = cacheObj.HandleSaveError(saveSC) + $$$ThrowOnError(status) + } +} + +/// Adds a module to the cache. Method AddCacheItem( pModuleFileName As %String, pSubDirectory As %String, @@ -191,33 +273,18 @@ Method AddCacheItem( set pName = "" set pVersionString = "" try { - // Get list of what's in module.xml - set tSC = $system.OBJ.Load(pModuleFileName,"-d",,.tLoadedList,1) - if $$$ISERR(tSC) { - // Wasn't a valid file. We'll just continue. - set tSC = $$$OK - quit - } - - if ($length(tLoadedList,",") > 1) { - // Contained multiple documents - tricky! We'll just continue. - quit - } - - set tExt = $zconvert($piece($get(tLoadedList),".",*),"U") - if (tExt '= "ZPM") { - quit + // Parse the module file + kill tStream, tName, tVersionString + try { + do ..ParseModuleFile(pModuleFileName, ..Root, .pName, .pVersionString, .tStream) + } catch ex { + // Wasn't a valid file or failed to parse. Log as warning and continue + set tSC = ex.AsStatus() + do ##class(%IPM.General.LogManager).Warning("Failed to parse module manifest in "_pModuleFileName_": "_$system.Status.GetErrorText(tSC),1) + return $$$OK } - kill tStream,tName,tVersionString - set tParseSC = ..GetModuleStreamFromFile(pModuleFileName,.tStream,.pName,.pVersionString) - if $$$ISERR(tParseSC) { - // Log as a warning, but keep going. - do ##class(%IPM.General.LogManager).Warning("Failed to parse module manifest in "_pModuleFileName_": "_$system.Status.GetErrorText(tParseSC),1) - quit - } - - // Create cache item. + // Create cache item if ##class(%IPM.Repo.Filesystem.Cache).CacheItemIndexExists(..Root,pSubDirectory) { set tCacheItem = ##class(%IPM.Repo.Filesystem.Cache).CacheItemIndexOpen(..Root,pSubDirectory,,.tSC) $$$ThrowOnError(tSC) @@ -228,6 +295,7 @@ Method AddCacheItem( } set tCacheItem.Name = pName set tCacheItem.VersionString = pVersionString + set tCacheItem.FileMTime = $zdatetime(##class(%Library.File).GetFileDateModified(pModuleFileName), 3) do tCacheItem.Manifest.CopyFrom(tStream) set tCacheItem.LastModified = $zdatetime($ztimestamp,3) set tSaveSC = tCacheItem.%Save() @@ -378,6 +446,9 @@ Storage Default Depth + +CacheLastRebuilt + FilesystemRepoDefinitionDefaultData %Storage.Persistent diff --git a/src/cls/IPM/Repo/Filesystem/PackageService.cls b/src/cls/IPM/Repo/Filesystem/PackageService.cls index f2476ae9e..1e00d2a70 100644 --- a/src/cls/IPM/Repo/Filesystem/PackageService.cls +++ b/src/cls/IPM/Repo/Filesystem/PackageService.cls @@ -24,9 +24,10 @@ Method HasModule(pModuleReference As %IPM.Storage.ModuleInfo) As %Boolean Method GetModuleManifest(pModuleReference As %IPM.Storage.ModuleInfo) As %Stream.Object { - set tModule = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..Root,pModuleReference.Name,pModuleReference.VersionString,,.tStatus) - $$$ThrowOnError(tStatus) - quit tModule.Manifest + // Use validated open to ensure cache is fresh + set module = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpenValidated(..Root, pModuleReference.Name, pModuleReference.VersionString, , .status) + $$$ThrowOnError(status) + quit module.Manifest } Method GetModule( @@ -47,11 +48,11 @@ Method GetModule( Method GetModuleDirectory(pModuleReference As %IPM.Storage.ModuleInfo) As %String { - // Get the module ... - set tModule = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..Root,pModuleReference.Name,pModuleReference.VersionString,,.tStatus) - $$$ThrowOnError(tStatus) + // Use validated open to ensure cache is fresh + set module = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpenValidated(..Root, pModuleReference.Name, pModuleReference.VersionString, , .status) + $$$ThrowOnError(status) - quit ##class(%File).NormalizeDirectory(tModule.Root_tModule.SubDirectory) + quit ##class(%File).NormalizeDirectory(module.Root_module.SubDirectory) } /// Returns 1 if the service supports a particular method. From a27e7e3d97f6ee9816d111504cdc607e0041dec3 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 18 Feb 2026 13:49:53 -0500 Subject: [PATCH 2/8] Add timing metrics --- src/cls/IPM/Main.cls | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index 16d00a36f..d31a05017 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -2158,10 +2158,11 @@ ClassMethod Repository(ByRef pCommandInfo) [ Internal ] // Rebuild cache write !,"Rebuilding cache for repository: ",repoName + set start = $zhorolog set sc = serverDef.BuildCache(1, 1) // purge=1, verbose=1 $$$ThrowOnError(sc) - write !,"Cache rebuilt successfully" + write !,"Cache rebuilt successfully in ", $zhorolog-start, " seconds." } else { set tName = $$$GetModifier(pCommandInfo,"name") set tType = $listget(serverClassList) @@ -2268,7 +2269,7 @@ ClassMethod ShowModulesForRepository( set server = ##class(%IPM.Repo.Definition).ServerDefinitionKeyOpen(pRepoName,,.tSC) $$$ThrowOnError(tSC) if server.%IsA("%IPM.Repo.Filesystem.Definition") { - write !,"Last full cache rebuild for '", pRepoName, "' was on ", server.CacheLastRebuilt, " (UTC). Run 'repo -n "_pRepoName_" -rebuild-cache' to rebuild the cache.",! + write !,"Last full cache rebuild for '", pRepoName, "' was on ", server.CacheLastRebuilt, " (UTC). Run 'repo -n "_pRepoName_" -rebuild-cache' to refresh the cache.",! } } From 1fff7f030d459a3b72602d34e3b2e7622c1f74df Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 18 Feb 2026 16:27:57 -0500 Subject: [PATCH 3/8] Add lazy loading of module manifests for speed --- src/cls/IPM/Repo/Filesystem/Cache.cls | 6 ++ src/cls/IPM/Repo/Filesystem/Definition.cls | 64 ++++++++++++++----- .../IPM/Repo/Filesystem/PackageService.cls | 14 ++++ 3 files changed, 69 insertions(+), 15 deletions(-) diff --git a/src/cls/IPM/Repo/Filesystem/Cache.cls b/src/cls/IPM/Repo/Filesystem/Cache.cls index 68891ce9b..4dba58c54 100644 --- a/src/cls/IPM/Repo/Filesystem/Cache.cls +++ b/src/cls/IPM/Repo/Filesystem/Cache.cls @@ -18,6 +18,9 @@ Property LastModified As %TimeStamp [ Required ]; /// Filesystem modification time of the module.xml file Property FileMTime As %TimeStamp; +/// Indicates if full manifest has been populated (deferred loading optimization) +Property ManifestLoaded As %Boolean [ InitialExpression = 0 ]; + /// Full module manifest Property Manifest As %Stream.GlobalCharacter; @@ -238,6 +241,9 @@ Storage Default FileMTime + +ManifestLoaded + ^IPM.Repo.Filesystem.CacheD CacheDefaultData diff --git a/src/cls/IPM/Repo/Filesystem/Definition.cls b/src/cls/IPM/Repo/Filesystem/Definition.cls index 5e9e4b696..4d64e31cf 100644 --- a/src/cls/IPM/Repo/Filesystem/Definition.cls +++ b/src/cls/IPM/Repo/Filesystem/Definition.cls @@ -231,9 +231,9 @@ Method ParseModuleFile( $$$ThrowStatus($$$ERROR($$$GeneralError, "File is not a ZPM module")) } - // Parse module info + // Parse module info (skip full manifest) kill stream, name, versionString - $$$ThrowOnError(..GetModuleStreamFromFile(filePath, .stream, .name, .versionString)) + $$$ThrowOnError(..GetModuleStreamFromFile(filePath, .stream, .name, .versionString, 1)) } /// Refresh a cache entry by re-parsing its module.xml file @@ -243,7 +243,7 @@ Method RefreshCacheEntry( filePath As %String, newMTime As %TimeStamp) { - // Parse the module file + // Parse the module file (metadata only) kill stream, name, versionString do ..ParseModuleFile(filePath, cacheObj.Root, .name, .versionString, .stream) @@ -251,8 +251,7 @@ Method RefreshCacheEntry( set cacheObj.Name = name set cacheObj.VersionString = versionString set cacheObj.FileMTime = newMTime - do cacheObj.Manifest.Clear() - do cacheObj.Manifest.CopyFrom(stream) + set cacheObj.ManifestLoaded = 0 // Mark manifest as not loaded (lazy load on next access) set cacheObj.LastModified = $zdatetime($ztimestamp, 3) set saveSC = cacheObj.%Save() @@ -273,7 +272,7 @@ Method AddCacheItem( set pName = "" set pVersionString = "" try { - // Parse the module file + // Parse the module file (metadata only) kill tStream, tName, tVersionString try { do ..ParseModuleFile(pModuleFileName, ..Root, .pName, .pVersionString, .tStream) @@ -296,7 +295,7 @@ Method AddCacheItem( set tCacheItem.Name = pName set tCacheItem.VersionString = pVersionString set tCacheItem.FileMTime = $zdatetime(##class(%Library.File).GetFileDateModified(pModuleFileName), 3) - do tCacheItem.Manifest.CopyFrom(tStream) + set tCacheItem.ManifestLoaded = 0 // Manifest will be loaded on load/install command set tCacheItem.LastModified = $zdatetime($ztimestamp,3) set tSaveSC = tCacheItem.%Save() if $$$ISERR(tSaveSC) { @@ -308,11 +307,36 @@ Method AddCacheItem( quit tSC } +/// Load the full manifest for a cache entry on-demand +/// Used for lazy loading optimization - only extracts manifest when needed +Method LoadManifestForCacheEntry( + cacheObj As %IPM.Repo.Filesystem.Cache, + filePath As %String) As %Status +{ + set sc = $$$OK + try { + // Extract full manifest using GetModuleStreamFromFile (not metadata-only) + kill stream, name, version + $$$ThrowOnError(..GetModuleStreamFromFile(filePath, .stream, .name, .version, 0)) + + // Update cache entry with manifest + do cacheObj.Manifest.Clear() + do cacheObj.Manifest.CopyFrom(stream) + set cacheObj.ManifestLoaded = 1 + set cacheObj.LastModified = $zdatetime($ztimestamp, 3) + $$$ThrowOnError(cacheObj.%Save()) + } catch e { + set sc = e.AsStatus() + } + quit sc +} + Method GetModuleStreamFromFile( pFilename As %String, Output pStream As %Stream.GlobalCharacter, Output pName As %String, - Output pVersion As %String) As %Status + Output pVersion As %String, + pMetadataOnly As %Boolean = 0) As %Status { set tSC = $$$OK set pName = "" @@ -323,18 +347,28 @@ Method GetModuleStreamFromFile( set tSC = tSourceStream.LinkToFile(pFilename) $$$ThrowOnError(tSC) - set pStream = ##class(%Stream.GlobalCharacter).%New() + // Copy to in-memory stream for reuse (avoid file I/O on rewind) + set tMemoryStream = ##class(%Stream.GlobalCharacter).%New() + set tSC = tMemoryStream.CopyFrom(tSourceStream) + $$$ThrowOnError(tSC) + do tMemoryStream.Rewind() + set tMetaStream = ##class(%Stream.GlobalCharacter).%New() - // Extract section of document - set tCompiledTransform = ##class(%IPM.Repo.XSLTProvider).GetCompiledTransformForXData($classname(),"ModuleDocumentTransform") - set tSC = ##class(%XML.XSLT.Transformer).TransformStreamWithCompiledXSL(tSourceStream,tCompiledTransform,.pStream) - $$$ThrowOnError(tSC) + // Extract section of document (skip if metadata-only) + if 'pMetadataOnly { + set pStream = ##class(%Stream.GlobalCharacter).%New() + set tCompiledTransform = ##class(%IPM.Repo.XSLTProvider).GetCompiledTransformForXData($classname(),"ModuleDocumentTransform") + set tSC = ##class(%XML.XSLT.Transformer).TransformStreamWithCompiledXSL(tMemoryStream,tCompiledTransform,.pStream) + $$$ThrowOnError(tSC) + + // Rewind in-memory stream for next transform + do tMemoryStream.Rewind() + } // Extract Name and Version - do tSourceStream.Rewind() set tCompiledTransform = ##class(%IPM.Repo.XSLTProvider).GetCompiledTransformForXData($classname(),"MetadataExtractionTransform") - set tSC = ##class(%XML.XSLT.Transformer).TransformStreamWithCompiledXSL(tSourceStream,tCompiledTransform,.tMetaStream) + set tSC = ##class(%XML.XSLT.Transformer).TransformStreamWithCompiledXSL(tMemoryStream,tCompiledTransform,.tMetaStream) $$$ThrowOnError(tSC) set tMetaStream.LineTerminator = $select($$$isWINDOWS:$char(13,10),$$$isUNIX:$char(10)) diff --git a/src/cls/IPM/Repo/Filesystem/PackageService.cls b/src/cls/IPM/Repo/Filesystem/PackageService.cls index 1e00d2a70..84a9505fd 100644 --- a/src/cls/IPM/Repo/Filesystem/PackageService.cls +++ b/src/cls/IPM/Repo/Filesystem/PackageService.cls @@ -27,6 +27,20 @@ Method GetModuleManifest(pModuleReference As %IPM.Storage.ModuleInfo) As %Stream // Use validated open to ensure cache is fresh set module = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpenValidated(..Root, pModuleReference.Name, pModuleReference.VersionString, , .status) $$$ThrowOnError(status) + + // Lazy-load manifest if not already loaded + if 'module.ManifestLoaded { + set defObj = ##class(%IPM.Repo.Filesystem.Definition).RootIndexOpen(module.Root, , .status) + $$$ThrowOnError(status) + + // Reconstruct file path + set dirPath = ##class(%File).NormalizeFilename(module.SubDirectory, module.Root) + set filePath = ##class(%File).NormalizeFilename("module.xml", dirPath) + + set status = defObj.LoadManifestForCacheEntry(module, filePath) + $$$ThrowOnError(status) + } + quit module.Manifest } From 1218436154a00aa806fc2651d3ee9c9fbbe38256 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 19 Feb 2026 13:37:10 -0500 Subject: [PATCH 4/8] Filesystem repo speed ups with Python and selectively skipping validation --- src/cls/IPM/Main.cls | 2 +- src/cls/IPM/Repo/Filesystem/Cache.cls | 6 + src/cls/IPM/Repo/Filesystem/Definition.cls | 258 ++++++++++++++++++--- 3 files changed, 232 insertions(+), 34 deletions(-) diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index d31a05017..6b58e4a25 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -2159,7 +2159,7 @@ ClassMethod Repository(ByRef pCommandInfo) [ Internal ] // Rebuild cache write !,"Rebuilding cache for repository: ",repoName set start = $zhorolog - set sc = serverDef.BuildCache(1, 1) // purge=1, verbose=1 + set sc = serverDef.BuildCache(0, 1) // purge=0, verbose=1 $$$ThrowOnError(sc) write !,"Cache rebuilt successfully in ", $zhorolog-start, " seconds." diff --git a/src/cls/IPM/Repo/Filesystem/Cache.cls b/src/cls/IPM/Repo/Filesystem/Cache.cls index 4dba58c54..527413e4e 100644 --- a/src/cls/IPM/Repo/Filesystem/Cache.cls +++ b/src/cls/IPM/Repo/Filesystem/Cache.cls @@ -21,6 +21,9 @@ Property FileMTime As %TimeStamp; /// Indicates if full manifest has been populated (deferred loading optimization) Property ManifestLoaded As %Boolean [ InitialExpression = 0 ]; +/// Indicates if file has been validated as a valid ZPM document +Property ValidationPassed As %Boolean [ InitialExpression = 0 ]; + /// Full module manifest Property Manifest As %Stream.GlobalCharacter; @@ -244,6 +247,9 @@ Storage Default ManifestLoaded + +ValidationPassed + ^IPM.Repo.Filesystem.CacheD CacheDefaultData diff --git a/src/cls/IPM/Repo/Filesystem/Definition.cls b/src/cls/IPM/Repo/Filesystem/Definition.cls index 4d64e31cf..89fb27fc1 100644 --- a/src/cls/IPM/Repo/Filesystem/Definition.cls +++ b/src/cls/IPM/Repo/Filesystem/Definition.cls @@ -127,6 +127,12 @@ Trigger DepthChanged [ Event = UPDATE, Foreach = row/object ] set oldDepth = {Depth*O} set newDepth = {Depth} + // Skip if we're already in the middle of BuildCache (e.g., auto-detecting depth) + // This prevents infinite recursion when BuildCache sets the depth + if $get(^||IPMBuildingCache) { + quit + } + // Depth changed - rebuild cache set oref = ..%Open({ID}) if $isobject(oref) { @@ -138,12 +144,16 @@ Trigger DepthChanged [ Event = UPDATE, Foreach = row/object ] /// Build the cache of modules found in the filesystem repository. /// pPurge - if 1, deletes existing cache entries before rebuilding Method BuildCache( - pPurge As %Boolean = 1, + pPurge As %Boolean = 0, pVerbose As %Integer = 0, pAutoDetectDepth As %Boolean = 0) As %Status { set tSC = $$$OK set tInitTLevel = $tlevel + + // Set flag to prevent DepthChanged trigger from recursively calling BuildCache + set ^||IPMBuildingCache = 1 + try { set tLogManager = ##class(%IPM.General.LogManager).%Get(.tSC) $$$ThrowOnError(tSC) @@ -153,6 +163,10 @@ Method BuildCache( $$$ThrowOnError(tSC) tstart + + // Track visited cache entries (for cleanup of stale entries) + kill tVisitedIds + if (pPurge) && (..%Id() '= "") { set tLockManager = ##class(%IPM.Utils.LockManager).%New() $$$ThrowOnError(tLockManager.LockClassId($classname(),..%Id())) @@ -171,7 +185,12 @@ Method BuildCache( } // Scan root directory recursively, up to ..Depth levels down, for module.xml files. - set tSC = ..ScanDirectory(..Root,.tFilenameList,,..Depth,$select(pVerbose>1:1,1:0),.tMaxDepth) + // Try Python scanner first (much faster), fall back to SQL-based scanner if unavailable + set tSC = ..ScanDirectoryPython(..Root,.tFilenameList,..Depth,$select(pVerbose>1:1,1:0),.tMaxDepth) + if $$$ISERR(tSC) { + write:pVerbose !,"Python scanner not available, using SQL-based scanner. ", $system.Status.GetErrorText(tSC) + set tSC = ..ScanDirectory(..Root,.tFilenameList,,..Depth,$select(pVerbose>1:1,1:0),.tMaxDepth) + } if $$$ISERR(tSC) { quit } @@ -192,11 +211,23 @@ Method BuildCache( quit:(tKey="") set tSubDirectory = tFilenameList(tKey,"sub") - set tAggSC = $$$ADDSC(tAggSC,..AddCacheItem(tFile,tSubDirectory,.tName,.tVersionString)) + set tAggSC = $$$ADDSC(tAggSC,..AddCacheItem(tFile,tSubDirectory,.tName,.tVersionString,.tCacheId)) + + // Track this cache entry as visited (still exists on filesystem) + if tCacheId '= "" { + set tVisitedIds(tCacheId) = 1 + } + write:pVerbose !,tName," ",tVersionString," @ ",##class(%File).NormalizeDirectory(..Root_tSubDirectory) } $$$ThrowOnError(tAggSC) + // Cleanup stale entries (files deleted from filesystem) + if 'pPurge { + set tSC = ..CleanupStaleEntries(.tVisitedIds, pVerbose) + $$$ThrowOnError(tSC) + } + // set cache rebuild timestamp using UTC set ..CacheLastRebuilt = $zdatetime($ztimestamp,3) $$$ThrowOnError(..%Save()) @@ -205,6 +236,10 @@ Method BuildCache( set tSC = e.AsStatus() } while ($tlevel > tInitTLevel) { trollback 1 } + + // Done building cache so clear the flag + set ^||IPMBuildingCache = 0 + quit tSC } @@ -214,24 +249,28 @@ Method ParseModuleFile( root As %String, Output name As %String, Output versionString As %String, - Output stream As %Stream.GlobalCharacter) + Output stream As %Stream.GlobalCharacter, + skipValidation As %Boolean = 0) { set name = "" set versionString = "" - // Validate file is a valid Studio document export - $$$ThrowOnError($system.OBJ.Load(filePath, "-d", , .loadedList, 1)) + // Only validate if not skipping (file already validated and unchanged) + if 'skipValidation { + // Validate file is a valid Studio document export + $$$ThrowOnError($system.OBJ.Load(filePath, "-d", , .loadedList, 1)) - if ($length(loadedList, ",") > 1) { - $$$ThrowStatus($$$ERROR($$$GeneralError, "File contains multiple documents")) - } + if ($length(loadedList, ",") > 1) { + $$$ThrowStatus($$$ERROR($$$GeneralError, "File contains multiple documents")) + } - set ext = $zconvert($piece($get(loadedList), ".", *), "U") - if (ext '= "ZPM") { - $$$ThrowStatus($$$ERROR($$$GeneralError, "File is not a ZPM module")) + set ext = $zconvert($piece($get(loadedList), ".", *), "U") + if (ext '= "ZPM") { + $$$ThrowStatus($$$ERROR($$$GeneralError, "File is not a ZPM module")) + } } - // Parse module info (skip full manifest) + // Parse module info (skip full manifest if metadata-only) kill stream, name, versionString $$$ThrowOnError(..GetModuleStreamFromFile(filePath, .stream, .name, .versionString, 1)) } @@ -243,21 +282,22 @@ Method RefreshCacheEntry( filePath As %String, newMTime As %TimeStamp) { - // Parse the module file (metadata only) + // Parse the module file (metadata only, re-validate since file changed) kill stream, name, versionString - do ..ParseModuleFile(filePath, cacheObj.Root, .name, .versionString, .stream) + do ..ParseModuleFile(filePath, cacheObj.Root, .name, .versionString, .stream, 0) // Update the cache entry set cacheObj.Name = name set cacheObj.VersionString = versionString set cacheObj.FileMTime = newMTime set cacheObj.ManifestLoaded = 0 // Mark manifest as not loaded (lazy load on next access) + set cacheObj.ValidationPassed = 1 // Mark validation passed set cacheObj.LastModified = $zdatetime($ztimestamp, 3) - set saveSC = cacheObj.%Save() - if $$$ISERR(saveSC) { - set status = cacheObj.HandleSaveError(saveSC) - $$$ThrowOnError(status) + set sc = cacheObj.%Save() + if $$$ISERR(sc) { + set sc = cacheObj.HandleSaveError(sc) + $$$ThrowOnError(sc) } } @@ -266,47 +306,105 @@ Method AddCacheItem( pModuleFileName As %String, pSubDirectory As %String, Output pName As %String, - Output pVersionString As %String) As %Status + Output pVersionString As %String, + Output pCacheId As %String = "") As %Status { set tSC = $$$OK set pName = "" set pVersionString = "" + set pCacheId = "" try { - // Parse the module file (metadata only) - kill tStream, tName, tVersionString - try { - do ..ParseModuleFile(pModuleFileName, ..Root, .pName, .pVersionString, .tStream) - } catch ex { - // Wasn't a valid file or failed to parse. Log as warning and continue - set tSC = ex.AsStatus() - do ##class(%IPM.General.LogManager).Warning("Failed to parse module manifest in "_pModuleFileName_": "_$system.Status.GetErrorText(tSC),1) - return $$$OK - } + // Check if cache entry exists and validation already passed + set tCurrentMTime = $zdatetime(##class(%Library.File).GetFileDateModified(pModuleFileName), 3) + set skipValidation = 0 - // Create cache item if ##class(%IPM.Repo.Filesystem.Cache).CacheItemIndexExists(..Root,pSubDirectory) { set tCacheItem = ##class(%IPM.Repo.Filesystem.Cache).CacheItemIndexOpen(..Root,pSubDirectory,,.tSC) $$$ThrowOnError(tSC) + + // Check if we can skip validation (file unchanged and previously validated) + set skipValidation = (tCacheItem.FileMTime = tCurrentMTime) && tCacheItem.ValidationPassed } else { set tCacheItem = ##class(%IPM.Repo.Filesystem.Cache).%New() set tCacheItem.Root = ..Root set tCacheItem.SubDirectory = pSubDirectory } + + // Parse the module file (metadata only, with conditional validation) + kill tStream, tName, tVersionString + try { + do ..ParseModuleFile(pModuleFileName, ..Root, .pName, .pVersionString, .tStream, skipValidation) + } catch ex { + // Wasn't a valid file or failed to parse. Log as warning and continue + set tSC = ex.AsStatus() + do ##class(%IPM.General.LogManager).Warning("Failed to parse module manifest in "_pModuleFileName_": "_$system.Status.GetErrorText(tSC),1) + return $$$OK + } + + // Update cache item set tCacheItem.Name = pName set tCacheItem.VersionString = pVersionString - set tCacheItem.FileMTime = $zdatetime(##class(%Library.File).GetFileDateModified(pModuleFileName), 3) - set tCacheItem.ManifestLoaded = 0 // Manifest will be loaded on load/install command + set tCacheItem.FileMTime = tCurrentMTime + set tCacheItem.ManifestLoaded = 0 // Manifest will be loaded on-demand + set tCacheItem.ValidationPassed = 1 // Mark validation passed set tCacheItem.LastModified = $zdatetime($ztimestamp,3) set tSaveSC = tCacheItem.%Save() if $$$ISERR(tSaveSC) { set tSC = tCacheItem.HandleSaveError(tSaveSC) } + + // Return cache ID for tracking + set pCacheId = tCacheItem.%Id() } catch e { set tSC = e.AsStatus() } quit tSC } +/// Delete cache entries for files that no longer exist on filesystem +Method CleanupStaleEntries( + ByRef visitedIds, + verbose As %Integer = 0) As %Status +{ + set sc = $$$OK + try { + // Query all cache entries for this root + set stmt = ##class(%SQL.Statement).%New() + set sc = stmt.%Prepare("SELECT ID, Name, VersionString, SubDirectory FROM %IPM_Repo_Filesystem.Cache WHERE Root = ?") + $$$ThrowOnError(sc) + + set result = stmt.%Execute(..Root) + set deletedCount = 0 + + while result.%Next() { + set cacheId = result.%Get("ID") + + // If not visited during scan, delete it (file was removed) + if '$data(visitedIds(cacheId)) { + set name = result.%Get("Name") + set version = result.%Get("VersionString") + set subDir = result.%Get("SubDirectory") + + write:verbose !,"Removing stale cache entry: ",name," ",version," (file deleted from ",..Root,subDir,")" + + set delSC = ##class(%IPM.Repo.Filesystem.Cache).%DeleteId(cacheId) + if $$$ISERR(delSC) { + set sc = $$$ADDSC(sc, delSC) + } else { + set deletedCount = deletedCount + 1 + } + } + } + + if (deletedCount > 0) && verbose { + write !,"Cleaned up ",deletedCount," stale cache entries" + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + /// Load the full manifest for a cache entry on-demand /// Used for lazy loading optimization - only extracts manifest when needed Method LoadManifestForCacheEntry( @@ -411,6 +509,100 @@ XData MetadataExtractionTransform } +/// Fast directory scanning using embedded Python - returns dynamic object with results +/// Uses Python's os.walk() which is C-based and highly optimized +ClassMethod ScanDirectoryPythonImpl( + root As %String, + depth As %Integer = 0, + verbose As %Boolean = 0) As %DynamicObject [ Language = python ] +{ +import os +import iris + +# Track results and max depth +results = [] +max_depth = 0 + +try: + for dirpath, dirnames, filenames in os.walk(root): + if 'module.xml' in filenames: + full_path = os.path.join(dirpath, 'module.xml') + + # Calculate relative directory from root + if dirpath == root or dirpath == root.rstrip(os.sep): + rel_dir = '' + else: + rel_dir = os.path.relpath(dirpath, root) + # Convert backslashes to forward slashes for consistency + rel_dir = rel_dir.replace('\\', '/') + + # Track max depth + if rel_dir: + current_depth = rel_dir.count('/') + 1 + if current_depth > max_depth: + max_depth = current_depth + + # Store result as dict + results.append({'fullPath': full_path, 'relDir': rel_dir}) + + if verbose: + print(f"Found file: {full_path}") + + # Depth limiting + if depth > 0: + current_depth = dirpath[len(root):].lstrip(os.sep).count(os.sep) + if current_depth >= depth: + dirnames.clear() # Don't descend further + + # Return plain dict + return { + 'results': results, + 'maxDepth': max_depth + } + +except Exception as e: + error_msg = f"Error scanning directory with Python: {str(e)}" + print(error_msg) + raise +} + +/// Wrapper method that calls Python scanner and converts results to ObjectScript array +ClassMethod ScanDirectoryPython( + root As %String, + ByRef filenameList, + depth As %Integer = 0, + verbose As %Boolean = 0, + Output maxDepth As %Integer = 0) As %Status +{ + set sc = $$$OK + try { + // Normalize root path + set root = ##class(%File).NormalizeDirectory(root) + + // Call Python implementation + set result = ..ScanDirectoryPythonImpl(root, depth, verbose) + + // Extract results and max depth from Python dict + set resultsList = result."__getitem__"("results") + set maxDepth = result."__getitem__"("maxDepth") + + // Convert Python list to ObjectScript array + set filenameList = 0 + set listLen = resultsList."__len__"() + for idx = 0:1:(listLen-1) { + set item = resultsList."__getitem__"(idx) + set fullPath = item."__getitem__"("fullPath") + set relDir = item."__getitem__"("relDir") + + set filenameList($increment(filenameList)) = fullPath + set filenameList(filenameList, "sub") = relDir + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + ClassMethod ScanDirectory( pRoot As %String, ByRef pFilenameList, From e8308e228147dd40fcae236aa89365e1aea68225 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 5 Mar 2026 11:23:15 -0500 Subject: [PATCH 5/8] Add tests and update changelog --- CHANGELOG.md | 3 +- src/cls/IPM/Repo/Filesystem/Definition.cls | 32 +- .../Test/PM/Integration/FilesystemRepo.cls | 296 ++++++++++++++++++ .../module-shallow/1.0.0/module.xml | 10 + .../nested/module-deep/1.0.0/module.xml | 10 + .../fs-cache-test/module-a/1.0.0/module.xml | 10 + .../fs-cache-test/module-b/1.0.0/module.xml | 10 + .../fs-cache-test/module-b/2.0.0/module.xml | 10 + 8 files changed, 369 insertions(+), 12 deletions(-) create mode 100644 tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/module-shallow/1.0.0/module.xml create mode 100644 tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/nested/module-deep/1.0.0/module.xml create mode 100644 tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-a/1.0.0/module.xml create mode 100644 tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/1.0.0/module.xml create mode 100644 tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/2.0.0/module.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 6929969ac..626aac485 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,16 +10,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - #992: Implement automatic history purge logic - #973: Enables CORS and JWT configuration for WebApplications in module.xml +- #536: Improve filesystem repository cache building through performance improvements, better staleness checking based on mtime, and new `-rebuild-cache` flag for `repo` command to manually rebuild the entire cache ### Fixed - #1001: The `unmap` and `enable` commands will now only activate CPF merge once after all namespaces have been configured instead after every namespace - #1052: In a namespace with mapped IPM, the `info` command works again and the intro message displays the IPM version and where its mapped from + ## [0.10.6] - 2026-02-24 ### Added - #1024: Added flag -export-python-deps to publish command -- #536: When installing from a file system repos, if the cached version of the module is stale (the modification time of the `module.xml` is different than when it was cached), pull again from disk. Also add `-rebuild-cache` flag for the `repo` command has been to rebuild the file system repo's entire cache. ### Fixed - #996: Ensure COS commands execute in exec under a dedicated, isolated context diff --git a/src/cls/IPM/Repo/Filesystem/Definition.cls b/src/cls/IPM/Repo/Filesystem/Definition.cls index 89fb27fc1..e87cf9026 100644 --- a/src/cls/IPM/Repo/Filesystem/Definition.cls +++ b/src/cls/IPM/Repo/Filesystem/Definition.cls @@ -85,8 +85,8 @@ ClassMethod OnConfigure( set pInstance.Depth = tDepth } - // This also saves it. - $$$ThrowOnError(pInstance.BuildCache(1,1,1)) + // Build cache without auto-detect - user specifies depth explicitly or it defaults to 0 (unlimited) + $$$ThrowOnError(pInstance.BuildCache(1,1,0)) } catch e { set tSC = e.AsStatus() } @@ -525,7 +525,23 @@ max_depth = 0 try: for dirpath, dirnames, filenames in os.walk(root): + # Calculate current depth before processing + # Must compute each iteration because os.walk() provides absolute paths, not incremental navigation + # We derive depth by: removing root prefix, stripping separators, then counting remaining separators + 1 + if dirpath == root or dirpath == root.rstrip(os.sep): + current_depth = 0 + else: + current_depth = dirpath[len(root):].lstrip(os.sep).count(os.sep) + 1 + + # Depth limiting - stop descending if we've reached the limit + if depth > 0 and current_depth >= depth: + dirnames.clear() # Don't descend further + if 'module.xml' in filenames: + # Skip if this directory exceeds the depth limit + if depth > 0 and current_depth > depth: + continue + full_path = os.path.join(dirpath, 'module.xml') # Calculate relative directory from root @@ -538,9 +554,9 @@ try: # Track max depth if rel_dir: - current_depth = rel_dir.count('/') + 1 - if current_depth > max_depth: - max_depth = current_depth + file_depth = rel_dir.count('/') + 1 + if file_depth > max_depth: + max_depth = file_depth # Store result as dict results.append({'fullPath': full_path, 'relDir': rel_dir}) @@ -548,12 +564,6 @@ try: if verbose: print(f"Found file: {full_path}") - # Depth limiting - if depth > 0: - current_depth = dirpath[len(root):].lstrip(os.sep).count(os.sep) - if current_depth >= depth: - dirnames.clear() # Don't descend further - # Return plain dict return { 'results': results, diff --git a/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls b/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls new file mode 100644 index 000000000..6c1064a7f --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls @@ -0,0 +1,296 @@ +Class Test.PM.Integration.FilesystemRepo Extends Test.PM.Integration.Base +{ + +Parameter REPONAME = "fs-cache-test"; + +Property RepoPath As %String; + +XData ModuleB300 +{ + + + + + module-b + 3.0.0 + module + + + +} + +Method OnBeforeAllTests() As %Status +{ + // Set up the repo path - use GetModuleDir utility + set ..RepoPath = ..GetModuleDir("fs-cache-test") + + // Create filesystem repo pointing to test data + set sc = ##class(%IPM.Main).Shell("repo -n "_..#REPONAME_" -fs -path "_..RepoPath) + do $$$AssertStatusOK(sc,"Created fs-cache-test repo successfully.") + quit sc +} + +Method OnAfterAllTests() As %Status +{ + // Remove test repository + set sc = ##class(%IPM.Main).Shell("repo -delete -name "_..#REPONAME) + do $$$AssertStatusOK(sc,"Deleted fs-cache-test repo successfully.") + quit sc +} + +Method OnAfterOneTest() As %Status +{ + // Uninstall all modules after each test for clean state + set sc = ##class(%IPM.Main).Shell("uninstall -all") + do $$$AssertStatusOK(sc,"Uninstalled all modules after test.") + quit sc +} + +/// Test that cache is built correctly when repo is created +Method Test01CacheBuiltCorrectly() +{ + // Get the repo definition + set repoDef = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen(..#REPONAME,,.sc) + do $$$AssertStatusOK(sc,"Opened repo definition") + do $$$AssertTrue($isobject(repoDef),"Repo definition exists") + + // Verify cache entries exist for both modules + set cacheA = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc) + do $$$AssertStatusOK(sc,"Cache entry for module-a 1.0.0 exists") + do $$$AssertTrue($isobject(cacheA),"module-a 1.0.0 found in cache") + + // Verify initial cache state - manifest not loaded, but validation passed during cache build + // ManifestLoaded=0 means only metadata (name/version) was extracted, not full manifest (lazy loading) + // ValidationPassed=1 means the module.xml was validated as a valid ZPM document during cache build + do $$$AssertEquals(cacheA.ManifestLoaded,0,"ManifestLoaded is initially 0 (lazy loading)") + do $$$AssertEquals(cacheA.ValidationPassed,1,"ValidationPassed is 1 after cache build") + + set cacheB1 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","1.0.0",,.sc) + do $$$AssertStatusOK(sc,"Cache entry for module-b 1.0.0 exists") + do $$$AssertTrue($isobject(cacheB1),"module-b 1.0.0 found in cache") + + set cacheB2 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","2.0.0",,.sc) + do $$$AssertStatusOK(sc,"Cache entry for module-b 2.0.0 exists") + do $$$AssertTrue($isobject(cacheB2),"module-b 2.0.0 found in cache") + + // Verify CacheLastRebuilt is set + do $$$AssertNotEquals(repoDef.CacheLastRebuilt,"","CacheLastRebuilt timestamp is set") + + // Test installing a module to verify cache works + set sc = ##class(%IPM.Main).Shell("install module-a 1.0.0") + do $$$AssertStatusOK(sc,"Successfully installed module-a from cache") +} + +/// Test lazy loading - manifests are not fully loaded until accessed +Method Test02LazyLoadingVerification() +{ + // Rebuild cache to ensure clean state with ManifestLoaded=0 + set sc = ##class(%IPM.Main).Shell("repo -n "_..#REPONAME_" -rebuild-cache") + do $$$AssertStatusOK(sc,"Rebuilt cache for clean state") + + // Get cache entry + set cacheA = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc) + do $$$AssertStatusOK(sc,"Opened cache entry for module-a") + + // Verify ManifestLoaded flag is initially false (lazy loading optimization) + do $$$AssertEquals(cacheA.ManifestLoaded,0,"ManifestLoaded flag is initially false") + + // Accessing basic metadata should work without loading full manifest + do $$$AssertEquals(cacheA.Name,"module-a","Can access Name without full manifest load") + do $$$AssertEquals(cacheA.VersionString,"1.0.0","Can access Version without full manifest load") + + // Now trigger full manifest load by accessing the manifest + set repoDef = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen(..#REPONAME,,.sc) + do $$$AssertStatusOK(sc,"Opened repo definition") + + set packageService = repoDef.GetPackageService() + + // Create ModuleInfo object for GetModuleManifest call + set moduleInfo = ##class(%IPM.Storage.ModuleInfo).%New() + set moduleInfo.Name = "module-a" + set moduleInfo.VersionString = "1.0.0" + + set manifest = packageService.GetModuleManifest(moduleInfo) + do $$$AssertTrue($isobject(manifest),"Got module manifest") + + // After accessing manifest, the ManifestLoaded flag should be set + set cacheAReloaded = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc) + do $$$AssertEquals(cacheAReloaded.ManifestLoaded,1,"ManifestLoaded flag is now true after accessing manifest") +} + +/// Test manual cache rebuild with -rebuild-cache command +Method Test03RebuildCache() +{ + // Get initial cache rebuild timestamp + set repoDef = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen(..#REPONAME,,.sc) + do $$$AssertStatusOK(sc,"Opened repo definition") + set initialTimestamp = repoDef.CacheLastRebuilt + + // Rebuild cache using command + set sc = ##class(%IPM.Main).Shell("repo -n "_..#REPONAME_" -rebuild-cache") + do $$$AssertStatusOK(sc,"Cache rebuild command executed successfully") + + // Verify timestamp was updated + set repoDefAfter = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen(..#REPONAME,,.sc) + do $$$AssertStatusOK(sc,"Opened repo definition after rebuild") + do $$$AssertNotEquals(repoDefAfter.CacheLastRebuilt,initialTimestamp,"CacheLastRebuilt timestamp was updated") + + // Verify cache entries still exist + set cacheA = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc) + do $$$AssertStatusOK(sc,"Cache entry for module-a still exists after rebuild") + + // Test installing a module to verify cache still works + set sc = ##class(%IPM.Main).Shell("install module-b 2.0.0") + do $$$AssertStatusOK(sc,"Successfully installed module-b after cache rebuild") +} + +/// Test cache freshness detection via FileMTime tracking +Method Test04CacheFreshnessDetection() +{ + // Get the original cache entry + set cacheA = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc) + do $$$AssertStatusOK(sc,"Opened original cache entry") + set originalMTime = cacheA.FileMTime + do $$$AssertNotEquals(originalMTime,"","Original FileMTime is set") + + // Modify the module.xml file by rewriting it (updating mtime) + set moduleFile = ##class(%File).NormalizeFilename("module.xml",##class(%File).NormalizeFilename("module-a/1.0.0",..RepoPath)) + + // Read and rewrite the file to update its modification time + set file = ##class(%Stream.FileCharacter).%New() + set sc = file.LinkToFile(moduleFile) + do $$$AssertStatusOK(sc,"Linked to module file") + set content = "" + while 'file.AtEnd { + set content = content_file.Read() + } + do file.Clear() + set sc = file.LinkToFile(moduleFile) + do file.Write(content) + set sc = file.%Save() + do $$$AssertStatusOK(sc,"Updated module file mtime") + + // Access the cache entry via validated open (this checks FileMTime) + set cacheARefreshed = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpenValidated(..RepoPath,"module-a","1.0.0",,.sc) + do $$$AssertStatusOK(sc,"Cache entry was refreshed based on FileMTime") + + // Verify FileMTime was updated + do $$$AssertNotEquals(cacheARefreshed.FileMTime,originalMTime,"FileMTime was updated after file modification") + + // Verify we can still install the module + set sc = ##class(%IPM.Main).Shell("install module-a 1.0.0") + do $$$AssertStatusOK(sc,"Successfully installed module after cache refresh") +} + +/// Test adding a new version to the filesystem +Method Test05AddNewVersion() +{ + // Create new directory for module-b 3.0.0 + set newVersionDir = ##class(%File).NormalizeFilename("module-b/3.0.0",..RepoPath) + set created = ##class(%File).CreateDirectory(newVersionDir) + do $$$AssertTrue(created,"Created directory for module-b 3.0.0") + + // Create module.xml file from XData block + set moduleFile = ##class(%File).NormalizeFilename("module.xml",newVersionDir) + + // Read XData content + set xdata = ##class(%Dictionary.XDataDefinition).%OpenId($classname()_"||ModuleB300",,.sc) + do $$$AssertStatusOK(sc,"Opened XData block") + + set file = ##class(%Stream.FileCharacter).%New() + set sc = file.LinkToFile(moduleFile) + do $$$AssertStatusOK(sc,"Linked to new module file") + do file.CopyFrom(xdata.Data) + set sc = file.%Save() + do $$$AssertStatusOK(sc,"Saved new module.xml file") + + // Rebuild cache to pick up new version + set sc = ##class(%IPM.Main).Shell("repo -n "_..#REPONAME_" -rebuild-cache") + do $$$AssertStatusOK(sc,"Rebuilt cache") + + // Verify new version is in cache + set cacheB3 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","3.0.0",,.sc) + do $$$AssertStatusOK(sc,"Cache entry for module-b 3.0.0 exists") + do $$$AssertTrue($isobject(cacheB3),"module-b 3.0.0 found in cache") + + // Test installing the new version + set sc = ##class(%IPM.Main).Shell("install module-b 3.0.0") + do $$$AssertStatusOK(sc,"Successfully installed module-b 3.0.0") +} + +/// Test deleting a version and verifying stale entry cleanup +Method Test06DeleteVersionStaleCleanup() +{ + // Verify module-b 3.0.0 exists in cache (from previous test) + set cacheB3 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","3.0.0",,.sc) + do $$$AssertTrue($isobject(cacheB3),"module-b 3.0.0 exists before deletion") + + // Delete the directory + set versionDir = ##class(%File).NormalizeFilename("module-b/3.0.0",..RepoPath) + set deleted = ##class(%File).RemoveDirectoryTree(versionDir) + do $$$AssertTrue(deleted,"Deleted module-b 3.0.0 directory") + + // Rebuild cache to trigger cleanup + set sc = ##class(%IPM.Main).Shell("repo -n "_..#REPONAME_" -rebuild-cache") + do $$$AssertStatusOK(sc,"Rebuilt cache after deletion") + + // Verify cache entry was removed + set cacheB3After = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","3.0.0",,.sc) + do $$$AssertTrue('$isobject(cacheB3After),"module-b 3.0.0 removed from cache after deletion") + + // Verify other versions still exist + set cacheB2 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","2.0.0",,.sc) + do $$$AssertTrue($isobject(cacheB2),"module-b 2.0.0 still exists in cache") +} + +/// Test depth specification - create repo with depth and verify it only scans to that depth +Method Test07DepthSpecification() +{ + // Test structure: + // fs-cache-test-depth/module-shallow/1.0.0/module.xml (2 levels deep) + // fs-cache-test-depth/nested/module-deep/1.0.0/module.xml (3 levels deep) + + // Create a new repo with depth=2 (should only find modules up to 2 levels deep) + set depthRepoPath = ..GetModuleDir("fs-cache-test-depth") + set sc = ##class(%IPM.Main).Shell("repo -n fs-cache-depth -fs -path "_depthRepoPath_" -depth 2") + do $$$AssertStatusOK(sc,"Created repo with depth=2") + + // Get repo definition and verify depth is set + set repoDef = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen("fs-cache-depth",,.sc) + do $$$AssertStatusOK(sc,"Opened depth repo definition") + do $$$AssertEquals(repoDef.Depth,2,"Depth is set to 2") + + // Verify that only modules at depth 2 are cached (not nested deeper) + // module-shallow at depth 2 should be found + set cacheShallow = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(depthRepoPath,"module-shallow","1.0.0",,.sc) + do $$$AssertTrue($isobject(cacheShallow),"module-shallow at depth 2 found in cache") + + // module-deep at depth 3 should NOT be found (exceeds depth limit) + set cacheDeep = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(depthRepoPath,"module-deep","1.0.0",,.sc) + do $$$AssertTrue('$isobject(cacheDeep),"module-deep at depth 3 not found in cache (exceeds depth)") + + // Clean up depth repo + set sc = ##class(%IPM.Main).Shell("repo -delete -name fs-cache-depth") + do $$$AssertStatusOK(sc,"Cleaned up depth test repo") + + // Test repo without depth specification (depth=0, unlimited) + set sc = ##class(%IPM.Main).Shell("repo -n fs-cache-unlimited -fs -path "_depthRepoPath) + do $$$AssertStatusOK(sc,"Created repo with unlimited depth") + + set repoDefUnlimited = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen("fs-cache-unlimited",,.sc) + do $$$AssertStatusOK(sc,"Opened unlimited depth repo definition") + do $$$AssertEquals(repoDefUnlimited.Depth,0,"Depth is 0 (unlimited)") + + // Both shallow and deep modules should be found with unlimited depth + set cacheShallow2 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(depthRepoPath,"module-shallow","1.0.0",,.sc) + do $$$AssertTrue($isobject(cacheShallow2),"module-shallow found with unlimited depth") + + set cacheDeep2 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(depthRepoPath,"module-deep","1.0.0",,.sc) + do $$$AssertTrue($isobject(cacheDeep2),"module-deep found with unlimited depth") + + // Clean up + set sc = ##class(%IPM.Main).Shell("repo -delete -name fs-cache-unlimited") + do $$$AssertStatusOK(sc,"Cleaned up unlimited depth test repo") +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/module-shallow/1.0.0/module.xml b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/module-shallow/1.0.0/module.xml new file mode 100644 index 000000000..53d76dddd --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/module-shallow/1.0.0/module.xml @@ -0,0 +1,10 @@ + + + + + module-shallow + 1.0.0 + module + + + diff --git a/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/nested/module-deep/1.0.0/module.xml b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/nested/module-deep/1.0.0/module.xml new file mode 100644 index 000000000..ffceef4de --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/nested/module-deep/1.0.0/module.xml @@ -0,0 +1,10 @@ + + + + + module-deep + 1.0.0 + module + + + diff --git a/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-a/1.0.0/module.xml b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-a/1.0.0/module.xml new file mode 100644 index 000000000..d2f4b2c41 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-a/1.0.0/module.xml @@ -0,0 +1,10 @@ + + + + + module-a + 1.0.0 + module + + + diff --git a/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/1.0.0/module.xml b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/1.0.0/module.xml new file mode 100644 index 000000000..e97ae98fb --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/1.0.0/module.xml @@ -0,0 +1,10 @@ + + + + + module-b + 1.0.0 + module + + + diff --git a/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/2.0.0/module.xml b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/2.0.0/module.xml new file mode 100644 index 000000000..e9726bf90 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/2.0.0/module.xml @@ -0,0 +1,10 @@ + + + + + module-b + 2.0.0 + module + + + From 78cab09fe8b8aa1d5df1f3cdddee130626911686 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Fri, 6 Mar 2026 13:14:58 -0500 Subject: [PATCH 6/8] Auto-rebuild cache on install and skip parsing entirely for unchanged modules --- CHANGELOG.md | 2 +- src/cls/IPM/Main.cls | 29 ++++++++++++- src/cls/IPM/Repo/Filesystem/Definition.cls | 43 ++++++++++--------- .../Test/PM/Integration/FilesystemRepo.cls | 24 +++++++---- 4 files changed, 65 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 626aac485..7ff3c17ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - #992: Implement automatic history purge logic - #973: Enables CORS and JWT configuration for WebApplications in module.xml -- #536: Improve filesystem repository cache building through performance improvements, better staleness checking based on mtime, and new `-rebuild-cache` flag for `repo` command to manually rebuild the entire cache +- #536: Improve filesystem repository cache through performance improvements, smart auto-cache rebuilding on install, and new `-rebuild-cache` flag for `repo` command to manually rebuild the entire cache ### Fixed - #1001: The `unmap` and `enable` commands will now only activate CPF merge once after all namespaces have been configured instead after every namespace diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index 6b58e4a25..ac6307ca2 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -2156,10 +2156,10 @@ ClassMethod Repository(ByRef pCommandInfo) [ Internal ] $$$ThrowStatus($$$ERROR($$$GeneralError,"Cache rebuild only supported for filesystem repositories")) } - // Rebuild cache + // Rebuild cache with purge write !,"Rebuilding cache for repository: ",repoName set start = $zhorolog - set sc = serverDef.BuildCache(0, 1) // purge=0, verbose=1 + set sc = serverDef.BuildCache(1, 1) // purge=1, verbose=1 $$$ThrowOnError(sc) write !,"Cache rebuilt successfully in ", $zhorolog-start, " seconds." @@ -2457,6 +2457,31 @@ ClassMethod Install( $$$ThrowStatus($$$ERROR($$$GeneralError, "No repositories are configured and enabled in this namespace.")) } + // Rebuild cache for filesystem repositories before searching + // This ensures newly added modules are discovered + write !,"Rebuilding cache(s) for filesystem repo(s)..." + if (tRegistry '= "") { + // Specific repository requested - only rebuild that one if it's filesystem + set tRepoDef = ##class(%IPM.Repo.Definition).ServerDefinitionKeyOpen(tRegistry,,.tSC) + if $$$ISOK(tSC) && $isobject(tRepoDef) && tRepoDef.%IsA("%IPM.Repo.Filesystem.Definition") { + do tRepoDef.BuildCache(0, 0) // purge=0, verbose=0 + } + } else { + // No specific repo - rebuild all enabled filesystem repos + &sql(DECLARE fsRepoCursor CURSOR FOR + SELECT ID INTO :tRepoId FROM %IPM_Repo_Filesystem.Definition WHERE Enabled = 1) + &sql(OPEN fsRepoCursor) + &sql(FETCH fsRepoCursor) + while (SQLCODE = 0) { + set tFsRepo = ##class(%IPM.Repo.Filesystem.Definition).%OpenId(tRepoId,,.tSC) + if $$$ISOK(tSC) && $isobject(tFsRepo) { + do tFsRepo.BuildCache(0, 0) // purge=0, verbose=0 + } + &sql(FETCH fsRepoCursor) + } + &sql(CLOSE fsRepoCursor) + } + set tVersion = $get(pCommandInfo("parameters","version")) set tKeywords = $$$GetModifier(pCommandInfo,"keywords") set tForce = $$$HasModifier(pCommandInfo,"force") diff --git a/src/cls/IPM/Repo/Filesystem/Definition.cls b/src/cls/IPM/Repo/Filesystem/Definition.cls index e87cf9026..a66774bd8 100644 --- a/src/cls/IPM/Repo/Filesystem/Definition.cls +++ b/src/cls/IPM/Repo/Filesystem/Definition.cls @@ -248,26 +248,21 @@ Method ParseModuleFile( filePath As %String, root As %String, Output name As %String, - Output versionString As %String, - Output stream As %Stream.GlobalCharacter, - skipValidation As %Boolean = 0) + Output versionString As %String) { set name = "" set versionString = "" - // Only validate if not skipping (file already validated and unchanged) - if 'skipValidation { - // Validate file is a valid Studio document export - $$$ThrowOnError($system.OBJ.Load(filePath, "-d", , .loadedList, 1)) + // Validate file is a valid Studio document export + $$$ThrowOnError($system.OBJ.Load(filePath, "-d", , .loadedList, 1)) - if ($length(loadedList, ",") > 1) { - $$$ThrowStatus($$$ERROR($$$GeneralError, "File contains multiple documents")) - } + if ($length(loadedList, ",") > 1) { + $$$ThrowStatus($$$ERROR($$$GeneralError, "File contains multiple documents")) + } - set ext = $zconvert($piece($get(loadedList), ".", *), "U") - if (ext '= "ZPM") { - $$$ThrowStatus($$$ERROR($$$GeneralError, "File is not a ZPM module")) - } + set ext = $zconvert($piece($get(loadedList), ".", *), "U") + if (ext '= "ZPM") { + $$$ThrowStatus($$$ERROR($$$GeneralError, "File is not a ZPM module")) } // Parse module info (skip full manifest if metadata-only) @@ -284,7 +279,7 @@ Method RefreshCacheEntry( { // Parse the module file (metadata only, re-validate since file changed) kill stream, name, versionString - do ..ParseModuleFile(filePath, cacheObj.Root, .name, .versionString, .stream, 0) + do ..ParseModuleFile(filePath, cacheObj.Root, .name, .versionString) // Update the cache entry set cacheObj.Name = name @@ -314,26 +309,32 @@ Method AddCacheItem( set pVersionString = "" set pCacheId = "" try { - // Check if cache entry exists and validation already passed + // Check if cache entry exists set tCurrentMTime = $zdatetime(##class(%Library.File).GetFileDateModified(pModuleFileName), 3) - set skipValidation = 0 if ##class(%IPM.Repo.Filesystem.Cache).CacheItemIndexExists(..Root,pSubDirectory) { set tCacheItem = ##class(%IPM.Repo.Filesystem.Cache).CacheItemIndexOpen(..Root,pSubDirectory,,.tSC) $$$ThrowOnError(tSC) - // Check if we can skip validation (file unchanged and previously validated) - set skipValidation = (tCacheItem.FileMTime = tCurrentMTime) && tCacheItem.ValidationPassed + // Check if we can skip parsing entirely (file unchanged and previously validated) + if (tCacheItem.FileMTime = tCurrentMTime) && tCacheItem.ValidationPassed { + // File unchanged - reuse cached values without parsing or updating + set pName = tCacheItem.Name + set pVersionString = tCacheItem.VersionString + set pCacheId = tCacheItem.%Id() + return $$$OK + } } else { + // New cache entry - need to create, parse and validate set tCacheItem = ##class(%IPM.Repo.Filesystem.Cache).%New() set tCacheItem.Root = ..Root set tCacheItem.SubDirectory = pSubDirectory } - // Parse the module file (metadata only, with conditional validation) + // Parse the module file (metadata only, validate since file is new or changed) kill tStream, tName, tVersionString try { - do ..ParseModuleFile(pModuleFileName, ..Root, .pName, .pVersionString, .tStream, skipValidation) + do ..ParseModuleFile(pModuleFileName, ..Root, .pName, .pVersionString) } catch ex { // Wasn't a valid file or failed to parse. Log as warning and continue set tSC = ex.AsStatus() diff --git a/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls b/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls index 6c1064a7f..e0860be5b 100644 --- a/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls +++ b/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls @@ -126,6 +126,9 @@ Method Test03RebuildCache() do $$$AssertStatusOK(sc,"Opened repo definition") set initialTimestamp = repoDef.CacheLastRebuilt + // Wait a second to ensure timestamp difference + hang 1 + // Rebuild cache using command set sc = ##class(%IPM.Main).Shell("repo -n "_..#REPONAME_" -rebuild-cache") do $$$AssertStatusOK(sc,"Cache rebuild command executed successfully") @@ -182,7 +185,7 @@ Method Test04CacheFreshnessDetection() do $$$AssertStatusOK(sc,"Successfully installed module after cache refresh") } -/// Test adding a new version to the filesystem +/// Test adding a new version to the filesystem and auto-discovery on install Method Test05AddNewVersion() { // Create new directory for module-b 3.0.0 @@ -204,18 +207,21 @@ Method Test05AddNewVersion() set sc = file.%Save() do $$$AssertStatusOK(sc,"Saved new module.xml file") - // Rebuild cache to pick up new version - set sc = ##class(%IPM.Main).Shell("repo -n "_..#REPONAME_" -rebuild-cache") - do $$$AssertStatusOK(sc,"Rebuilt cache") + // NOTE: No manual cache rebuild here - install should auto-discover the new version + // This tests the auto-rebuild feature (cache rebuilt before dependency resolution) - // Verify new version is in cache + // Verify new version is NOT yet in cache set cacheB3 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","3.0.0",,.sc) - do $$$AssertStatusOK(sc,"Cache entry for module-b 3.0.0 exists") - do $$$AssertTrue($isobject(cacheB3),"module-b 3.0.0 found in cache") + do $$$AssertTrue('$isobject(cacheB3),"module-b 3.0.0 not yet in cache before install") - // Test installing the new version + // Install should auto-rebuild cache and find the new version set sc = ##class(%IPM.Main).Shell("install module-b 3.0.0") - do $$$AssertStatusOK(sc,"Successfully installed module-b 3.0.0") + do $$$AssertStatusOK(sc,"Successfully installed module-b 3.0.0 with auto-rebuild") + + // Verify new version is NOW in cache (after install auto-rebuilt) + set cacheB3After = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","3.0.0",,.sc) + do $$$AssertStatusOK(sc,"Cache entry for module-b 3.0.0 exists after install") + do $$$AssertTrue($isobject(cacheB3After),"module-b 3.0.0 found in cache after auto-rebuild") } /// Test deleting a version and verifying stale entry cleanup From ec5a53a8d9ba0eb5325ff8b4c1a83420ccb6ef08 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Fri, 13 Mar 2026 10:50:57 -0400 Subject: [PATCH 7/8] Minor refactor --- CHANGELOG.md | 1 - src/cls/IPM/Repo/Filesystem/Cache.cls | 11 +++----- src/cls/IPM/Repo/Filesystem/Definition.cls | 26 +++++++++++-------- .../IPM/Repo/Filesystem/PackageService.cls | 3 ++- .../Test/PM/Integration/FilesystemRepo.cls | 2 -- 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ff3c17ca..89d886716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - #1001: The `unmap` and `enable` commands will now only activate CPF merge once after all namespaces have been configured instead after every namespace - #1052: In a namespace with mapped IPM, the `info` command works again and the intro message displays the IPM version and where its mapped from - ## [0.10.6] - 2026-02-24 ### Added diff --git a/src/cls/IPM/Repo/Filesystem/Cache.cls b/src/cls/IPM/Repo/Filesystem/Cache.cls index 527413e4e..ff8fbbed5 100644 --- a/src/cls/IPM/Repo/Filesystem/Cache.cls +++ b/src/cls/IPM/Repo/Filesystem/Cache.cls @@ -18,12 +18,12 @@ Property LastModified As %TimeStamp [ Required ]; /// Filesystem modification time of the module.xml file Property FileMTime As %TimeStamp; -/// Indicates if full manifest has been populated (deferred loading optimization) +/// Indicates if full manifest has been populated (deferred loading optimization). +/// When 0: Only metadata (Name, Version) is cached; Manifest stream is empty. +/// When 1: Full XML has been loaded into Manifest property. +/// Use LoadManifestForCacheEntry() to populate on-demand when needed. Property ManifestLoaded As %Boolean [ InitialExpression = 0 ]; -/// Indicates if file has been validated as a valid ZPM document -Property ValidationPassed As %Boolean [ InitialExpression = 0 ]; - /// Full module manifest Property Manifest As %Stream.GlobalCharacter; @@ -247,9 +247,6 @@ Storage Default ManifestLoaded - -ValidationPassed - ^IPM.Repo.Filesystem.CacheD CacheDefaultData diff --git a/src/cls/IPM/Repo/Filesystem/Definition.cls b/src/cls/IPM/Repo/Filesystem/Definition.cls index a66774bd8..0f67f15c9 100644 --- a/src/cls/IPM/Repo/Filesystem/Definition.cls +++ b/src/cls/IPM/Repo/Filesystem/Definition.cls @@ -266,12 +266,14 @@ Method ParseModuleFile( } // Parse module info (skip full manifest if metadata-only) - kill stream, name, versionString $$$ThrowOnError(..GetModuleStreamFromFile(filePath, .stream, .name, .versionString, 1)) } -/// Refresh a cache entry by re-parsing its module.xml file -/// Updates the cache entry in-place with new manifest and mtime +/// Refresh a cache entry by re-parsing its module.xml file. +/// Called when FileMTime indicates the file has changed since last cache. +/// Updates cache entry in-place with new metadata and mtime. +/// NOTE: Sets ManifestLoaded=0 to defer full manifest parsing until needed. +/// The manifest will be loaded on-demand when GetModuleManifest() is called. Method RefreshCacheEntry( cacheObj As %IPM.Repo.Filesystem.Cache, filePath As %String, @@ -285,8 +287,7 @@ Method RefreshCacheEntry( set cacheObj.Name = name set cacheObj.VersionString = versionString set cacheObj.FileMTime = newMTime - set cacheObj.ManifestLoaded = 0 // Mark manifest as not loaded (lazy load on next access) - set cacheObj.ValidationPassed = 1 // Mark validation passed + set cacheObj.ManifestLoaded = 0 // Defer manifest parsing - will load on first GetModuleManifest() call set cacheObj.LastModified = $zdatetime($ztimestamp, 3) set sc = cacheObj.%Save() @@ -317,7 +318,7 @@ Method AddCacheItem( $$$ThrowOnError(tSC) // Check if we can skip parsing entirely (file unchanged and previously validated) - if (tCacheItem.FileMTime = tCurrentMTime) && tCacheItem.ValidationPassed { + if (tCacheItem.FileMTime = tCurrentMTime) { // File unchanged - reuse cached values without parsing or updating set pName = tCacheItem.Name set pVersionString = tCacheItem.VersionString @@ -332,7 +333,6 @@ Method AddCacheItem( } // Parse the module file (metadata only, validate since file is new or changed) - kill tStream, tName, tVersionString try { do ..ParseModuleFile(pModuleFileName, ..Root, .pName, .pVersionString) } catch ex { @@ -346,8 +346,7 @@ Method AddCacheItem( set tCacheItem.Name = pName set tCacheItem.VersionString = pVersionString set tCacheItem.FileMTime = tCurrentMTime - set tCacheItem.ManifestLoaded = 0 // Manifest will be loaded on-demand - set tCacheItem.ValidationPassed = 1 // Mark validation passed + set tCacheItem.ManifestLoaded = 0 // Defer manifest parsing - will load on first GetModuleManifest() call set tCacheItem.LastModified = $zdatetime($ztimestamp,3) set tSaveSC = tCacheItem.%Save() if $$$ISERR(tSaveSC) { @@ -406,8 +405,13 @@ Method CleanupStaleEntries( quit sc } -/// Load the full manifest for a cache entry on-demand -/// Used for lazy loading optimization - only extracts manifest when needed +/// Load the full manifest for a cache entry on-demand (lazy loading optimization). +/// This is called only when the manifest is actually needed (e.g., for module installation). +/// Extracts the full XML section and stores it in the Manifest stream. +/// Sets ManifestLoaded=1 to prevent redundant parsing on subsequent accesses. +/// +/// Performance: Skipping manifest load during cache build provides 30-50% speedup +/// for operations that only need metadata (list, search, version resolution). Method LoadManifestForCacheEntry( cacheObj As %IPM.Repo.Filesystem.Cache, filePath As %String) As %Status diff --git a/src/cls/IPM/Repo/Filesystem/PackageService.cls b/src/cls/IPM/Repo/Filesystem/PackageService.cls index 84a9505fd..374955981 100644 --- a/src/cls/IPM/Repo/Filesystem/PackageService.cls +++ b/src/cls/IPM/Repo/Filesystem/PackageService.cls @@ -28,7 +28,8 @@ Method GetModuleManifest(pModuleReference As %IPM.Storage.ModuleInfo) As %Stream set module = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpenValidated(..Root, pModuleReference.Name, pModuleReference.VersionString, , .status) $$$ThrowOnError(status) - // Lazy-load manifest if not already loaded + // Lazy load manifest if not already populated (ManifestLoaded=0) + // This defers expensive XML parsing until manifest is actually needed if 'module.ManifestLoaded { set defObj = ##class(%IPM.Repo.Filesystem.Definition).RootIndexOpen(module.Root, , .status) $$$ThrowOnError(status) diff --git a/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls b/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls index e0860be5b..c82a03826 100644 --- a/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls +++ b/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls @@ -61,9 +61,7 @@ Method Test01CacheBuiltCorrectly() // Verify initial cache state - manifest not loaded, but validation passed during cache build // ManifestLoaded=0 means only metadata (name/version) was extracted, not full manifest (lazy loading) - // ValidationPassed=1 means the module.xml was validated as a valid ZPM document during cache build do $$$AssertEquals(cacheA.ManifestLoaded,0,"ManifestLoaded is initially 0 (lazy loading)") - do $$$AssertEquals(cacheA.ValidationPassed,1,"ValidationPassed is 1 after cache build") set cacheB1 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","1.0.0",,.sc) do $$$AssertStatusOK(sc,"Cache entry for module-b 1.0.0 exists") From 21f15ea4c4da6af8e3542860f8da52e861870ada Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Mon, 16 Mar 2026 09:42:06 -0400 Subject: [PATCH 8/8] Address review comments --- src/cls/IPM/Main.cls | 2 +- src/cls/IPM/Repo/Filesystem/Cache.cls | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index ac6307ca2..32e181e5b 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -2269,7 +2269,7 @@ ClassMethod ShowModulesForRepository( set server = ##class(%IPM.Repo.Definition).ServerDefinitionKeyOpen(pRepoName,,.tSC) $$$ThrowOnError(tSC) if server.%IsA("%IPM.Repo.Filesystem.Definition") { - write !,"Last full cache rebuild for '", pRepoName, "' was on ", server.CacheLastRebuilt, " (UTC). Run 'repo -n "_pRepoName_" -rebuild-cache' to refresh the cache.",! + write !,"Last full cache rebuild for '", pRepoName, "' was on ", server.CacheLastRebuilt, " (UTC). If needed, run 'repo -n "_pRepoName_" -rebuild-cache' to refresh the cache.",! } } diff --git a/src/cls/IPM/Repo/Filesystem/Cache.cls b/src/cls/IPM/Repo/Filesystem/Cache.cls index ff8fbbed5..78629328b 100644 --- a/src/cls/IPM/Repo/Filesystem/Cache.cls +++ b/src/cls/IPM/Repo/Filesystem/Cache.cls @@ -139,10 +139,14 @@ ClassMethod RootNameVersionOpenValidated( try { // Open the cache entry set cacheObj = ..RootNameVersionOpen(root, name, versionString, concurrency, .status) - $$$ThrowOnError(status) + if $$$ISERR(status) { + quit + } set defObj = ##class(%IPM.Repo.Filesystem.Definition).RootIndexOpen(root, , .status) - $$$ThrowOnError(status) + if $$$ISERR(status) { + quit + } // Check for schema migration (old cache entries without FileMTime) set dirPath = ##class(%File).NormalizeFilename(cacheObj.SubDirectory, cacheObj.Root) @@ -153,7 +157,6 @@ ClassMethod RootNameVersionOpenValidated( set horologTime = ##class(%Library.File).GetFileDateModified(filePath) set currentMTime = $zdatetime(horologTime, 3) do defObj.RefreshCacheEntry(cacheObj, filePath, currentMTime) - $$$ThrowOnError(status) } else { // File doesn't exist - invalidate cache set status = $$$ERROR($$$GeneralError, "module.xml not found in " _ dirPath) @@ -178,9 +181,6 @@ ClassMethod RootNameVersionOpenValidated( // Compare cached mtime with current mtime if (cacheObj.FileMTime '= currentMTime) { do defObj.RefreshCacheEntry(cacheObj, filePath, currentMTime) - if $$$ISERR(status) { - quit - } } } catch ex { set status = ex.AsStatus()