Improve filesystem repo caching

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - #992: Implement automatic history purge logic
 - #973: Enables CORS and JWT configuration for WebApplications in module.xml
+- #536: Improve filesystem repository cache through performance improvements, smart auto-cache rebuilding on install, and new `-rebuild-cache` flag for `repo` command to manually rebuild the entire cache
 
 ### Fixed
 - #1001: The `unmap` and `enable` commands will now only activate CPF merge once after all namespaces have been configured instead after every namespace

src/cls/IPM/Main.cls

@@ -2146,6 +2146,23 @@ ClassMethod Repository(ByRef pCommandInfo) [ Internal ]
         set tType = "%IPM.Repo.Remote.Definition"
         $$$ThrowOnError($classmethod(tType,"Configure",1,.tModifiers,.tData))
         do ..Shell("repo -list")
+    } elseif $$$HasModifier(pCommandInfo,"rebuild-cache") {
+        // Validate repository exists and is filesystem type
+        set repoName = $$$GetModifier(pCommandInfo,"name")
+        set serverDef = ##class(%IPM.Repo.Definition).ServerDefinitionKeyOpen(repoName,,.sc)
+        $$$ThrowOnError(sc)
+
+        if 'serverDef.%IsA("%IPM.Repo.Filesystem.Definition") {
+            $$$ThrowStatus($$$ERROR($$$GeneralError,"Cache rebuild only supported for filesystem repositories"))
+        }
+
+        // Rebuild cache with purge
+        write !,"Rebuilding cache for repository: ",repoName
+        set start = $zhorolog
+        set sc = serverDef.BuildCache(1, 1)  // purge=1, verbose=1
+        $$$ThrowOnError(sc)
+
+        write !,"Cache rebuilt successfully in ", $zhorolog-start, " seconds."
     } else {
         set tName = $$$GetModifier(pCommandInfo,"name")
         set tType = $listget(serverClassList)
@@ -2247,6 +2264,13 @@ ClassMethod ShowModulesForRepository(
     set list("width") = width
     write !
     do ..DisplayModules(.list)
+
+    // If filesystem repo, show last cache rebuild time and command to rebuild it since there's no auto full cache rebuild mechanism (other than reconfiguring the repo entirely)
+    set server = ##class(%IPM.Repo.Definition).ServerDefinitionKeyOpen(pRepoName,,.tSC)
+    $$$ThrowOnError(tSC)
+    if server.%IsA("%IPM.Repo.Filesystem.Definition") {
+        write !,"Last full cache rebuild for '", pRepoName, "' was on ", server.CacheLastRebuilt, " (UTC). Run 'repo -n "_pRepoName_" -rebuild-cache' to refresh the cache.",!
+    }
 }
 
 Query SourceControlClasses() As %SQLQuery(ROWSPEC = "ID:%String,Name:%String") [ SqlProc ]
@@ -2433,6 +2457,31 @@ ClassMethod Install(
             $$$ThrowStatus($$$ERROR($$$GeneralError, "No repositories are configured and enabled in this namespace."))
         }
 
+        // Rebuild cache for filesystem repositories before searching
+        // This ensures newly added modules are discovered
+        write !,"Rebuilding cache(s) for filesystem repo(s)..."
+        if (tRegistry '= "") {
+            // Specific repository requested - only rebuild that one if it's filesystem
+            set tRepoDef = ##class(%IPM.Repo.Definition).ServerDefinitionKeyOpen(tRegistry,,.tSC)
+            if $$$ISOK(tSC) && $isobject(tRepoDef) && tRepoDef.%IsA("%IPM.Repo.Filesystem.Definition") {
+                do tRepoDef.BuildCache(0, 0)  // purge=0, verbose=0
+            }
+        } else {
+            // No specific repo - rebuild all enabled filesystem repos
+            &sql(DECLARE fsRepoCursor CURSOR FOR
+                 SELECT ID INTO :tRepoId FROM %IPM_Repo_Filesystem.Definition WHERE Enabled = 1)
+            &sql(OPEN fsRepoCursor)
+            &sql(FETCH fsRepoCursor)
+            while (SQLCODE = 0) {
+                set tFsRepo = ##class(%IPM.Repo.Filesystem.Definition).%OpenId(tRepoId,,.tSC)
+                if $$$ISOK(tSC) && $isobject(tFsRepo) {
+                    do tFsRepo.BuildCache(0, 0)  // purge=0, verbose=0
+                }
+                &sql(FETCH fsRepoCursor)
+            }
+            &sql(CLOSE fsRepoCursor)
+        }
+
         set tVersion = $get(pCommandInfo("parameters","version"))
         set tKeywords = $$$GetModifier(pCommandInfo,"keywords")
         set tForce = $$$HasModifier(pCommandInfo,"force")

src/cls/IPM/Repo/Filesystem/Cache.cls

@@ -15,6 +15,15 @@ Property SubDirectory As %String(MAXLEN = 260);
 
 Property LastModified As %TimeStamp [ Required ];
 
+/// Filesystem modification time of the module.xml file
+Property FileMTime As %TimeStamp;
+
+/// Indicates if full manifest has been populated (deferred loading optimization).
+/// When 0: Only metadata (Name, Version) is cached; Manifest stream is empty.
+/// When 1: Full <Module> XML has been loaded into Manifest property.
+/// Use LoadManifestForCacheEntry() to populate on-demand when needed.
+Property ManifestLoaded As %Boolean [ InitialExpression = 0 ];
+
 /// Full module manifest
 Property Manifest As %Stream.GlobalCharacter;
 
@@ -113,6 +122,74 @@ Method HandleSaveError(pSC As %Status) As %Status
     quit tSC
 }
 
+/// Open a cache entry and validate it against the filesystem
+/// If the cached entry is stale (FileMTime differs from actual file mtime),
+/// re-parse the module.xml and update the cache entry
+/// Returns the cache object (possibly refreshed) or throws error if not found
+ClassMethod RootNameVersionOpenValidated(
+	root As %String,
+	name As %String,
+	versionString As %String,
+	concurrency As %Integer = -1,
+	Output status As %Status) As %IPM.Repo.Filesystem.Cache
+{
+    set status = $$$OK
+    set cacheObj = ""
+
+    try {
+        // Open the cache entry
+        set cacheObj = ..RootNameVersionOpen(root, name, versionString, concurrency, .status)
+        $$$ThrowOnError(status)
+
+        set defObj = ##class(%IPM.Repo.Filesystem.Definition).RootIndexOpen(root, , .status)
+        $$$ThrowOnError(status)
+
+        // Check for schema migration (old cache entries without FileMTime)
+        set dirPath = ##class(%File).NormalizeFilename(cacheObj.SubDirectory, cacheObj.Root)
+        set filePath = ##class(%File).NormalizeFilename("module.xml", dirPath)
+        if (cacheObj.FileMTime = "") {
+            // Old cache entry - refresh it to populate FileMTime
+            if ##class(%File).Exists(filePath) {
+                set horologTime = ##class(%Library.File).GetFileDateModified(filePath)
+                set currentMTime = $zdatetime(horologTime, 3)
+                do defObj.RefreshCacheEntry(cacheObj, filePath, currentMTime)
+                $$$ThrowOnError(status)
+            } else {
+                // File doesn't exist - invalidate cache
+                set status = $$$ERROR($$$GeneralError, "module.xml not found in " _ dirPath)
+                do cacheObj.%DeleteId(cacheObj.%Id())
+                set cacheObj = ""
+                quit
+            }
+        }
+
+        // Stat the file to get current mtime
+        if '##class(%File).Exists(filePath) {
+            // File deleted from filesystem - invalidate cache
+            set status = $$$ERROR($$$GeneralError, "module.xml not found in " _ dirPath)
+            do cacheObj.%DeleteId(cacheObj.%Id())
+            set cacheObj = ""
+            quit
+        }
+
+        set horologTime = ##class(%Library.File).GetFileDateModified(filePath)
+        set currentMTime = $zdatetime(horologTime, 3)  // ODBC format
+
+        // Compare cached mtime with current mtime
+        if (cacheObj.FileMTime '= currentMTime) {
+            do defObj.RefreshCacheEntry(cacheObj, filePath, currentMTime)
+            if $$$ISERR(status) {
+                quit
+            }
+        }
+    } catch ex {
+        set status = ex.AsStatus()
+        set cacheObj = ""
+    }
+
+    quit cacheObj
+}
+
 Storage Default
 {
 <Data name="CacheDefaultData">
@@ -164,6 +241,12 @@ Storage Default
 <Value name="16">
 <Value>DisplayName</Value>
 </Value>
+<Value name="17">
+<Value>FileMTime</Value>
+</Value>
+<Value name="18">
+<Value>ManifestLoaded</Value>
+</Value>
 </Data>
 <DataLocation>^IPM.Repo.Filesystem.CacheD</DataLocation>
 <DefaultData>CacheDefaultData</DefaultData>