fix: infinispan store migrator

caching-service/build.gradle

@@ -73,7 +73,6 @@ dependencies {
     implementation libs.janino
     implementation libs.jakarta.servlet.api
     implementation libs.lettuce
-
     testImplementation(testFixtures(project(":apiml-common")))
 
     testImplementation libs.spring.boot.starter.test
@@ -108,7 +107,7 @@ bootRun {
         '--add-opens=java.base/javax.net.ssl=ALL-UNNAMED',
         '--add-opens=java.base/java.net=ALL-UNNAMED'
     ])
-    
+
     debugOptions {
         port = 5016
         suspend = false

caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/LazyCacheManager.java

@@ -27,17 +27,24 @@
 import org.infinispan.manager.CacheManagerInfo;
 import org.infinispan.manager.DefaultCacheManager;
 import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.persistence.spi.PersistenceException;
 import org.infinispan.remoting.transport.Address;
 import org.infinispan.stats.CacheContainerStats;
+import org.infinispan.tools.store.migrator.StoreMigrator;
 import org.springframework.boot.context.event.ApplicationReadyEvent;
 import org.springframework.context.event.EventListener;
 
 import javax.security.auth.Subject;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.CompletionStage;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Phaser;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -267,6 +274,9 @@ class CacheInitializer {
 
         private final Phaser threadCounter = new Phaser();
 
+        // avoids retrying migration forever if it doesn't fix the underlying issue
+        private final Set<String> migrationAttempted = ConcurrentHashMap.newKeySet();
+
         private DefaultCacheManager startDefaultCacheManager() {
             var defaultCacheManager = new DefaultCacheManager(cacheManagerConfig, false);
             try {
@@ -356,6 +366,86 @@ public DefaultCacheManager getDefaultCacheManager() {
             }
         }
 
+        /**
+         * Migrates the on-disk Soft Index File Store of a cache that failed to start, most likely because it
+         * was written by an older, incompatible Infinispan version. Source and target both live under the
+         * cache's own persistent folder so the swap below can put the migrated store back in the exact
+         * location the cache configuration expects.
+         */
+        private static void migrateLegacyStore(String cacheName) {
+            var cacheDir = Paths.get(InfinispanConfig.getRootFolder(), cacheName);
+            var migratedDir = Paths.get(InfinispanConfig.getRootFolder(), cacheName + "-migrated");
+            var legacyBackupDir = Paths.get(InfinispanConfig.getRootFolder(), cacheName + "-legacy");
+
+            log.info("Attempting migration of legacy store for cache {} at {}", cacheName, cacheDir);
+            try {
+                // clear any leftover from a previous failed attempt so the migrator starts from an empty target
+                deleteRecursively(migratedDir);
+                new StoreMigrator(migrationProperties(cacheName, cacheDir, migratedDir)).run();
+
+                // clear any leftover backup, otherwise the move below fails (target already exists)
+                deleteRecursively(legacyBackupDir);
+                Files.move(cacheDir, legacyBackupDir);
+                Files.move(migratedDir, cacheDir);
+                log.info("Migration of legacy store for cache {} is completed.", cacheName);
+            } catch (Exception e) {
+                log.error(
+                    "Migration failed for cache {}. The source Soft Index File Store could not be fully read " +
+                        "or the migrated store could not be put back in place. The store may be incomplete or " +
+                        "inconsistent; the original store at {} was left untouched.",
+                    cacheName, cacheDir, e
+                );
+            }
+        }
+
+        static Properties migrationProperties(String cacheName, Path sourceDir, Path targetDir) {
+            var properties = new Properties();
+
+            properties.setProperty("source.type", "SOFT_INDEX_FILE_STORE");
+            properties.setProperty("source.cache_name", cacheName);
+            properties.setProperty("source.location", sourceDir.resolve("data").toString());
+            properties.setProperty("source.index_location", sourceDir.resolve("index").toString());
+            properties.setProperty("source.version", "15");
+
+            properties.setProperty("target.type", "SOFT_INDEX_FILE_STORE");
+            properties.setProperty("target.cache_name", cacheName);
+            properties.setProperty("target.location", targetDir.resolve("data").toString());
+            properties.setProperty("target.index_location", targetDir.resolve("index").toString());
+
+            return properties;
+        }
+
+        /**
+         * A failed store start is only worth retrying after a migration when the root cause is a
+         * {@link PersistenceException} - the signature of a Soft Index File Store written by an older,
+         * incompatible marshaller (e.g. "Found an invalid protobuf tag..."). Any other cause (bad config,
+         * permissions, etc.) would not be fixed by migrating the store, so it's not worth attempting.
+         */
+        static boolean isLegacyStoreFailure(Throwable t) {
+            for (Throwable cause = t; cause != null; cause = cause.getCause()) {
+                if (cause instanceof PersistenceException) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        static void deleteRecursively(Path dir) throws IOException {
+            if (!Files.exists(dir)) {
+                return;
+            }
+            try (var paths = Files.walk(dir)) {
+                paths.sorted(java.util.Comparator.reverseOrder())
+                    .forEach(p -> {
+                        try {
+                            Files.delete(p);
+                        } catch (IOException e) {
+                            log.warn("Cannot delete {}", p, e);
+                        }
+                    });
+            }
+        }
+
         private boolean createCache(String cacheName, ConfigurationBuilder cacheBuilder) {
             var cacheConfig = cacheBuilder.build();
             log.debug("Initializing cache {} with config {}", cacheName, cacheConfig);
@@ -365,6 +455,17 @@ private boolean createCache(String cacheName, ConfigurationBuilder cacheBuilder)
                     .getOrCreateCache(cacheName, cacheConfig);
                 return true;
             } catch (CacheConfigurationException cce) {
+                // check the real cause first: migrationAttempted.add() has a side effect (marks the cache as
+                // attempted), so it must only run when the failure is actually worth migrating for - otherwise
+                // an unrelated error would burn the one-shot retry and a real legacy-store failure later
+                // wouldn't get a chance to be fixed
+                if (isLegacyStoreFailure(cce) && migrationAttempted.add(cacheName)) {
+                    log.warn("Cache {} failed to start due to an incompatible on-disk store, retrying once after migration", cacheName, cce);
+                    migrateLegacyStore(cacheName);
+                    // recurse once: migrationAttempted already contains cacheName, so a second failure falls
+                    // through to the existing fallback below instead of looping
+                    return createCache(cacheName, cacheBuilder);
+                }
                 log.warn("Error during initialization of cache {}", cacheName, cce);
                 try {
                     underInit.defineConfiguration(cacheName, cacheConfig);

caching-service/src/test/java/org/zowe/apiml/caching/service/infinispan/config/LazyCacheManagerTest.java

@@ -0,0 +1,101 @@
+/*
+ * This program and the accompanying materials are made available under the terms of the
+ * Eclipse Public License v2.0 which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-v20.html
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Copyright Contributors to the Zowe Project.
+ */
+
+package org.zowe.apiml.caching.service.infinispan.config;
+
+import org.infinispan.persistence.spi.PersistenceException;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.CompletionException;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class LazyCacheManagerTest {
+
+    @Nested
+    class IsLegacyStoreFailure {
+
+        @Test
+        void whenCauseChainContainsPersistenceException_thenReturnTrue() {
+            var cause = new PersistenceException("Found an invalid protobuf tag (1) having a field number smaller than 1");
+            var wrapper = new CompletionException(cause);
+
+            assertTrue(LazyCacheManager.CacheInitializer.isLegacyStoreFailure(wrapper));
+        }
+
+        @Test
+        void whenExceptionItselfIsPersistenceException_thenReturnTrue() {
+            assertTrue(LazyCacheManager.CacheInitializer.isLegacyStoreFailure(new PersistenceException("boom")));
+        }
+
+        @Test
+        void whenCauseChainHasNoPersistenceException_thenReturnFalse() {
+            var unrelated = new IllegalStateException("permission denied", new IOException("disk full"));
+
+            assertFalse(LazyCacheManager.CacheInitializer.isLegacyStoreFailure(unrelated));
+        }
+
+        @Test
+        void whenExceptionHasNoCause_thenReturnFalse() {
+            assertFalse(LazyCacheManager.CacheInitializer.isLegacyStoreFailure(new IllegalStateException("no cause here")));
+        }
+    }
+
+    @Nested
+    class MigrationProperties {
+
+        @Test
+        void whenCalled_thenBuildsSourceAndTargetProperties() {
+            var sourceDir = Path.of("caching-service", "localhost", "invalidatedJwtTokens");
+            var targetDir = Path.of("caching-service", "localhost", "invalidatedJwtTokens-migrated");
+
+            var properties = LazyCacheManager.CacheInitializer.migrationProperties("invalidatedJwtTokens", sourceDir, targetDir);
+
+            assertEquals("SOFT_INDEX_FILE_STORE", properties.getProperty("source.type"));
+            assertEquals("invalidatedJwtTokens", properties.getProperty("source.cache_name"));
+            assertEquals(sourceDir.resolve("data").toString(), properties.getProperty("source.location"));
+            assertEquals(sourceDir.resolve("index").toString(), properties.getProperty("source.index_location"));
+            assertEquals("15", properties.getProperty("source.version"));
+
+            assertEquals("SOFT_INDEX_FILE_STORE", properties.getProperty("target.type"));
+            assertEquals("invalidatedJwtTokens", properties.getProperty("target.cache_name"));
+            assertEquals(targetDir.resolve("data").toString(), properties.getProperty("target.location"));
+            assertEquals(targetDir.resolve("index").toString(), properties.getProperty("target.index_location"));
+        }
+    }
+
+    @Nested
+    class DeleteRecursively {
+
+        @Test
+        void whenDirHasNestedFiles_thenDeletesEverything(@TempDir Path tempDir) throws IOException {
+            var nested = tempDir.resolve("data/sub");
+            Files.createDirectories(nested);
+            Files.writeString(nested.resolve("file.txt"), "content");
+            Files.writeString(tempDir.resolve("data/other.txt"), "content");
+
+            LazyCacheManager.CacheInitializer.deleteRecursively(tempDir.resolve("data"));
+
+            assertFalse(Files.exists(tempDir.resolve("data")));
+        }
+
+        @Test
+        void whenDirDoesNotExist_thenDoesNothing(@TempDir Path tempDir) throws IOException {
+            var missing = tempDir.resolve("does-not-exist");
+
+            assertDoesNotThrow(() -> LazyCacheManager.CacheInitializer.deleteRecursively(missing));
+        }
+    }
+}

gradle/versions.gradle

@@ -192,6 +192,7 @@ dependencyResolutionManagement {
             library('infinispan_lock', 'org.infinispan', 'infinispan-clustered-lock').versionRef('infinispan')
             library('infinispan_jboss_marshalling', 'org.infinispan', 'infinispan-jboss-marshalling').versionRef('infinispan')
             library('infinispan_annotations', 'org.infinispan', 'infinispan-component-annotations').versionRef('infinispan')
+            library('infinispan_tools', 'org.infinispan', 'infinispan-tools').versionRef('infinispan')
             library('jackson_annotations', 'com.fasterxml.jackson.core', 'jackson-annotations').versionRef('jacksonAnnotations')
             library('jackson_core', 'com.fasterxml.jackson.core', 'jackson-core').versionRef('jacksonCore')
             library('jackson_databind', 'com.fasterxml.jackson.core', 'jackson-databind').versionRef('jacksonDatabind')
@@ -309,7 +310,7 @@ dependencyResolutionManagement {
             library('mock_server','org.mock-server','mockserver-netty').versionRef('mockServer')
 
 
-            bundle('infinispan', [ 'infinispan_spring_boot3_starter_embedded', 'infinispan_lock', 'infinispan_jboss_marshalling', 'infinispan_annotations' ])
+            bundle('infinispan', [ 'infinispan_spring_boot3_starter_embedded', 'infinispan_lock', 'infinispan_jboss_marshalling', 'infinispan_annotations', 'infinispan_tools' ])
             bundle('jaxb', ['jaxbApi', 'jaxbImpl'])
             bundle('modulith', ['modulith', 'modulith_core', 'jmolecules'])
         }

infinispan-store-migrator/README.md

@@ -0,0 +1,10 @@
+# Store migrator
+
+Poc to showcase migration of the only distributed cache:
+
+- `invalidatedJwtTokens`
+- `zoweCache`
+- `zoweInvalidatedTokenCache`
+
+using the StoreMigrator Utility class from Infinispan Tools.
+StoreMigrator uses the `migration.properties` files for the migration.

infinispan-store-migrator/build.gradle

@@ -0,0 +1,18 @@
+plugins {
+    id 'java'
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    compileOnly libs.lombok
+    implementation libs.logback.classic
+    annotationProcessor libs.lombok
+    implementation("org.infinispan:infinispan-tools:16.2.1")
+}
+
+test {
+    useJUnitPlatform()
+}

infinispan-store-migrator/config/migrator-invalidatedJwtTokens.properties

@@ -0,0 +1,16 @@
+source.type=SOFT_INDEX_FILE_STORE
+source.cache_name=invalidatedJwtTokens
+source.location=caching-service/localhost/invalidatedJwtTokens/data
+source.index_location=caching-service/localhost/invalidatedJwtTokens/index
+source.version=15
+#source.segment_count=256
+#source.marshaller.class=org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller
+#source.marshaller.allow-list.regexps=.*
+
+target.type=SOFT_INDEX_FILE_STORE
+#target.marshaller.class=org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller
+#target.marshaller.allow-list.regexps=.*
+target.cache_name=invalidatedJwtTokens
+target.location=caching-service/localhost_NEW/invalidatedJwtTokens/data
+target.index_location=caching-service/localhost_NEW/invalidatedJwtTokens/index
+#target.segment_count=256

infinispan-store-migrator/config/migrator-zoweCache.properties

@@ -0,0 +1,16 @@
+source.type=SOFT_INDEX_FILE_STORE
+source.cache_name=zoweCache
+source.location=caching-service/localhost/zoweCache/data
+source.index_location=caching-service/localhost/zoweCache/index
+source.version=15
+#source.segment_count=256
+#source.marshaller.class=org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller
+#source.marshaller.allow-list.regexps=.*
+
+target.type=SOFT_INDEX_FILE_STORE
+#target.marshaller.class=org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller
+#target.marshaller.allow-list.regexps=.*
+target.cache_name=zoweCache
+target.location=caching-service/localhost_NEW/zoweCache/data
+target.index_location=caching-service/localhost_NEW/zoweCache/index
+#target.segment_count=256

infinispan-store-migrator/config/migrator-zoweInvalidatedTokenCache.properties

@@ -0,0 +1,16 @@
+source.type=SOFT_INDEX_FILE_STORE
+source.cache_name=zoweInvalidatedTokenCache
+source.location=caching-service/localhost/zoweInvalidatedTokenCache/data
+source.index_location=caching-service/localhost/zoweInvalidatedTokenCache/index
+source.version=15
+#source.segment_count=256
+#source.marshaller.class=org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller
+#source.marshaller.allow-list.regexps=.*
+
+target.type=SOFT_INDEX_FILE_STORE
+#target.marshaller.class=org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller
+#target.marshaller.allow-list.regexps=.*
+target.cache_name=zoweInvalidatedTokenCache
+target.location=caching-service/localhost_NEW/zoweInvalidatedTokenCache/data
+target.index_location=caching-service/localhost_NEW/zoweInvalidatedTokenCache/index
+#target.segment_count=256

infinispan-store-migrator/src/main/java/org/zowe/apiml/Main.java

@@ -0,0 +1,40 @@
+/*
+ * This program and the accompanying materials are made available under the terms of the
+ * Eclipse Public License v2.0 which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-v20.html
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Copyright Contributors to the Zowe Project.
+ */
+
+package org.zowe.apiml;
+
+import lombok.extern.slf4j.Slf4j;
+import org.infinispan.tools.store.migrator.StoreMigrator;
+
+@Slf4j
+public class Main {
+    public static void main(String[] args) {
+       migrate("infinispan-store-migrator/config/migrator-invalidatedJwtTokens.properties");
+       migrate("infinispan-store-migrator/config/migrator-zoweCache.properties");
+       migrate("infinispan-store-migrator/config/migrator-zoweInvalidatedTokenCache.properties");
+    }
+
+    private static void migrate(String properties) {
+        log.info("Migrating {}...", properties);
+
+        try {
+            StoreMigrator.main(new String[]{properties});
+            log.info("Migration using {} is completed.", properties);
+        } catch (Exception e) {
+            log.error(
+                "Migration failed for {}. Continuing with the remaining cache stores. " +
+                    "The source Soft Index File Store could not be fully read. " +
+                    "The store may be incomplete or inconsistent.",
+                properties,
+                e
+            );
+        }
+    }
+}