Add db-tune ops subcommand + production RDS runbook

openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DbTuneIT.java

@@ -0,0 +1,197 @@
+/*
+ *  Copyright 2026 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.openmetadata.it.tests;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Map;
+import org.jdbi.v3.core.Jdbi;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
+import org.openmetadata.it.bootstrap.TestSuiteBootstrap;
+import org.openmetadata.service.jdbi3.locator.ConnectionType;
+import org.openmetadata.service.util.dbtune.Action;
+import org.openmetadata.service.util.dbtune.AutoTuner;
+import org.openmetadata.service.util.dbtune.DbTuneResult;
+import org.openmetadata.service.util.dbtune.MysqlAutoTuner;
+import org.openmetadata.service.util.dbtune.PostgresAutoTuner;
+import org.openmetadata.service.util.dbtune.TableRecommendation;
+
+/**
+ * End-to-end tests for {@link AutoTuner} against the live Testcontainers database.
+ *
+ * <p>The read-only tests ({@link #analyzeReturnsRecommendationsForKnownTables}, {@link
+ * #dryRunDoesNotMutateReloptions}) run against the real catalog tables that the IT bootstrap
+ * created via migrations.
+ *
+ * <p>Tests that exercise the write path ({@link #applyExecutesAndIsIdempotent}, {@link
+ * #analyzeOneRunsOnIsolatedTable}) deliberately use a private throwaway table — never a real
+ * catalog table. Reason: {@code ALTER TABLE} on a shared production table bumps MySQL's per-table
+ * metadata version, which invalidates JDBC prepared-statement caches across the whole
+ * Testcontainer. When that table has a {@code JSON} column (e.g. {@code entity_relationship}), the
+ * driver's re-prepared metadata sometimes returns the column type as {@code VARBINARY}, and
+ * subsequent {@code INSERT} statements fail with {@code "Cannot create a JSON value from a string
+ * with CHARACTER SET 'binary'"}. We saw this break {@code GlossaryTermRelationsIT},
+ * {@code DomainResourceIT}, and the lineage ITs in CI when an earlier version of this test applied
+ * settings to {@code entity_relationship}. The recommendations themselves are sound — the IT just
+ * cannot afford the side effect on a shared DB.
+ *
+ * <p>Sequential because {@code @BeforeEach} / {@code @AfterEach} create and drop the same isolated
+ * table by name; concurrent execution would race.
+ */
+@Execution(ExecutionMode.SAME_THREAD)
+class DbTuneIT {
+
+  /** Table created and dropped per test — never a catalog table. Safe blast radius. */
+  private static final String ISOLATED_TABLE = "dbtune_it_isolated_table";
+
+  /** A real catalog table used only by the read-only tests to assert against the live schema. */
+  private static final String READ_ONLY_PROBE_TABLE = "entity_relationship";
+
+  @BeforeEach
+  void createIsolatedTable() {
+    Jdbi jdbi = TestSuiteBootstrap.getJdbi();
+    ConnectionType connType = currentConnectionType();
+    jdbi.useHandle(
+        handle -> {
+          handle.execute("DROP TABLE IF EXISTS " + quoteIdent(connType, ISOLATED_TABLE));
+          if (connType == ConnectionType.POSTGRES) {
+            handle.execute(
+                "CREATE TABLE " + quoteIdent(connType, ISOLATED_TABLE) + " (id INT PRIMARY KEY)");
+          } else {
+            handle.execute(
+                "CREATE TABLE "
+                    + quoteIdent(connType, ISOLATED_TABLE)
+                    + " (id INT PRIMARY KEY) ENGINE=InnoDB");
+          }
+        });
+  }
+
+  @AfterEach
+  void dropIsolatedTable() {
+    Jdbi jdbi = TestSuiteBootstrap.getJdbi();
+    ConnectionType connType = currentConnectionType();
+    jdbi.useHandle(
+        handle -> handle.execute("DROP TABLE IF EXISTS " + quoteIdent(connType, ISOLATED_TABLE)));
+  }
+
+  @Test
+  void analyzeReturnsRecommendationsForKnownTables() {
+    AutoTuner tuner = currentTuner();
+    Jdbi jdbi = TestSuiteBootstrap.getJdbi();
+
+    DbTuneResult result = jdbi.withHandle(tuner::analyze);
+
+    assertNotNull(result);
+    assertNotNull(result.engineVersion());
+    assertFalse(result.tableRecommendations().isEmpty(), "Expected at least one recommendation");
+    assertTrue(
+        result.tableRecommendations().stream()
+            .anyMatch(r -> READ_ONLY_PROBE_TABLE.equals(r.tableName())),
+        READ_ONLY_PROBE_TABLE + " should be in the recommendations");
+  }
+
+  @Test
+  void applyExecutesAndIsIdempotent() {
+    AutoTuner tuner = currentTuner();
+    Jdbi jdbi = TestSuiteBootstrap.getJdbi();
+    ConnectionType connType = currentConnectionType();
+    TableRecommendation rec = recommendationForIsolatedTable(connType);
+
+    jdbi.useHandle(handle -> tuner.apply(handle, rec));
+
+    String built = tuner.buildAlterStatement(rec);
+    assertTrue(built.contains(ISOLATED_TABLE), "ALTER target table mismatch: " + built);
+
+    // Apply twice — second invocation must complete without throwing.
+    jdbi.useHandle(handle -> tuner.apply(handle, rec));
+  }
+
+  @Test
+  void analyzeOneRunsOnIsolatedTable() {
+    AutoTuner tuner = currentTuner();
+    Jdbi jdbi = TestSuiteBootstrap.getJdbi();
+
+    jdbi.useHandle(handle -> tuner.analyzeOne(handle, ISOLATED_TABLE));
+  }
+
+  @Test
+  void dryRunDoesNotMutateReloptions() {
+    AutoTuner tuner = currentTuner();
+    Jdbi jdbi = TestSuiteBootstrap.getJdbi();
+
+    Map<String, String> before = currentSettingsFor(tuner, jdbi, READ_ONLY_PROBE_TABLE);
+
+    DbTuneResult result = jdbi.withHandle(tuner::analyze);
+    assertNotNull(result);
+
+    Map<String, String> after = currentSettingsFor(tuner, jdbi, READ_ONLY_PROBE_TABLE);
+    assertEquals(before, after, "Analyze (dry-run) must not change table settings");
+  }
+
+  // ---- helpers ----
+
+  private AutoTuner currentTuner() {
+    return currentConnectionType() == ConnectionType.POSTGRES
+        ? new PostgresAutoTuner()
+        : new MysqlAutoTuner();
+  }
+
+  private ConnectionType currentConnectionType() {
+    return "mysql".equalsIgnoreCase(System.getProperty("databaseType", "postgres"))
+        ? ConnectionType.MYSQL
+        : ConnectionType.POSTGRES;
+  }
+
+  /**
+   * Builds a {@link TableRecommendation} pointing at {@link #ISOLATED_TABLE} with engine-appropriate
+   * settings. We construct it directly rather than going through {@code analyze()} because the
+   * isolated table is intentionally NOT in the static catalog — that's how we keep the apply path
+   * off shared production tables.
+   */
+  private TableRecommendation recommendationForIsolatedTable(final ConnectionType connType) {
+    Map<String, String> recommended =
+        connType == ConnectionType.POSTGRES
+            ? Map.of("autovacuum_vacuum_scale_factor", "0.05")
+            : Map.of("STATS_PERSISTENT", "1", "STATS_AUTO_RECALC", "1");
+    return new TableRecommendation(
+        ISOLATED_TABLE, Action.APPLY, 0L, 0L, Map.of(), recommended, "Isolated IT test table");
+  }
+
+  /**
+   * Re-runs analyze and projects out the {@link TableRecommendation#currentSettings()} for the
+   * named table. Going through the same code path that built the original recommendation keeps the
+   * assertion stable across either dialect's parsing rules.
+   */
+  private Map<String, String> currentSettingsFor(
+      final AutoTuner tuner, final Jdbi jdbi, final String tableName) {
+    return jdbi.withHandle(tuner::analyze).tableRecommendations().stream()
+        .filter(r -> tableName.equals(r.tableName()))
+        .findFirst()
+        .map(TableRecommendation::currentSettings)
+        .orElse(Map.of());
+  }
+
+  private static String quoteIdent(final ConnectionType connType, final String identifier) {
+    if (!identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) {
+      throw new IllegalArgumentException("Refusing unsafe identifier: " + identifier);
+    }
+    return connType == ConnectionType.POSTGRES ? "\"" + identifier + "\"" : "`" + identifier + "`";
+  }
+}

openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java

@@ -133,6 +133,12 @@
 import org.openmetadata.service.secrets.SecretsManagerUpdateService;
 import org.openmetadata.service.security.auth.SecurityConfigurationManager;
 import org.openmetadata.service.security.jwt.JWTTokenGenerator;
+import org.openmetadata.service.util.dbtune.AutoTuner;
+import org.openmetadata.service.util.dbtune.DbTuneReport;
+import org.openmetadata.service.util.dbtune.DbTuneResult;
+import org.openmetadata.service.util.dbtune.MysqlAutoTuner;
+import org.openmetadata.service.util.dbtune.PostgresAutoTuner;
+import org.openmetadata.service.util.dbtune.TableRecommendation;
 import org.openmetadata.service.util.jdbi.DatabaseAuthenticationProviderFactory;
 import org.openmetadata.service.util.jdbi.JdbiUtils;
 import org.slf4j.LoggerFactory;
@@ -175,9 +181,12 @@ public Integer call() {
             + "'drop-create', 'changelog', 'migrate', 'migrate-secrets', 'reindex', 'reembed', 'reindex-rdf', 'reindexdi', 'deploy-pipelines', "
             + "'dbServiceCleanup', 'relationshipCleanup', 'tagUsageCleanup', 'drop-indexes', 'remove-security-config', 'create-indexes', "
             + "'setOpenMetadataUrl', 'configureEmailSettings', 'get-security-config', 'update-security-config', 'install-app', 'delete-app', 'create-user', 'reset-password', "
-            + "'syncAlertOffset', 'analyze-tables', 'cleanup-flowable-history', 'regenerate-bot-tokens'");
+            + "'syncAlertOffset', 'analyze-tables', 'db-tune', 'cleanup-flowable-history', 'regenerate-bot-tokens'");
     LOG.info(
         "Use 'reindex --auto-tune' for automatic performance optimization based on cluster capabilities");
+    LOG.info(
+        "Use 'db-tune' for a per-table autovacuum / InnoDB stats tuning report; add --apply to "
+            + "execute the recommendations and --analyze to refresh planner stats on changed tables");
     LOG.info(
         "Use 'cleanup-flowable-history --delete --runtime-batch-size=1000 --history-batch-size=1000' for Flowable cleanup with custom options");
     LOG.info(
@@ -2469,6 +2478,111 @@ public Integer analyzeTables() {
     }
   }
 
+  @Command(
+      name = "db-tune",
+      description =
+          "Generate a per-table autovacuum / InnoDB stats tuning report and optionally apply it. "
+              + "Default mode is read-only — pass --apply to execute the ALTER TABLE statements "
+              + "and --analyze to refresh planner stats on changed tables.")
+  public Integer dbTune(
+      @Option(
+              names = {"--apply"},
+              defaultValue = "false",
+              description =
+                  "Apply the recommendations. Without this flag the command only prints the report.")
+          boolean apply,
+      @Option(
+              names = {"--yes", "-y"},
+              defaultValue = "false",
+              description = "Skip the interactive confirmation when applying.")
+          boolean skipPrompt,
+      @Option(
+              names = {"--analyze"},
+              defaultValue = "false",
+              description =
+                  "After --apply, run ANALYZE on each changed table so planner stats reflect the new settings.")
+          boolean runAnalyze) {
+    try {
+      parseConfig();
+      String driverClass = config.getDataSourceFactory().getDriverClass();
+      ConnectionType connType = ConnectionType.from(driverClass);
+      if (connType == null) {
+        LOG.error(
+            "db-tune does not support driver class '{}'. Only the bundled MySQL and PostgreSQL drivers are recognised.",
+            driverClass);
+        return 1;
+      }
+      AutoTuner tuner = autoTunerFor(connType);
+      DbTuneResult result = jdbi.withHandle(tuner::analyze);
+      LOG.info("\n{}", DbTuneReport.render(result));
+      if (!apply) {
+        return 0;
+      }
+      List<TableRecommendation> actionable = result.actionableRecommendations();
+      if (actionable.isEmpty()) {
+        if (result.tableRecommendations().isEmpty()) {
+          LOG.info("Nothing to apply — no tracked tables exist on this database.");
+        } else {
+          LOG.info(
+              "Nothing to apply — every tracked table already matches its recommended settings.");
+        }
+        return 0;
+      }
+      if (!skipPrompt && !confirmApply(tuner, actionable)) {
+        LOG.info("Operation cancelled.");
+        return 0;
+      }
+      applyRecommendations(tuner, actionable, runAnalyze);
+      return 0;
+    } catch (Exception e) {
+      LOG.error("db-tune failed due to ", e);
+      return 1;
+    }
+  }
+
+  private AutoTuner autoTunerFor(final ConnectionType connType) {
+    return switch (connType) {
+      case POSTGRES -> new PostgresAutoTuner();
+      case MYSQL -> new MysqlAutoTuner();
+    };
+  }
+
+  private boolean confirmApply(final AutoTuner tuner, final List<TableRecommendation> actionable) {
+    LOG.info("About to apply {} ALTER statements:", actionable.size());
+    LOG.info("\n{}", DbTuneReport.renderAlterStatements(tuner, actionable));
+    @SuppressWarnings("resource")
+    Scanner scanner = new Scanner(System.in);
+    LOG.info("Apply now? [y/N]: ");
+    String input = scanner.hasNext() ? scanner.next().trim().toLowerCase() : "";
+    return input.equals("y") || input.equals("yes");
+  }
+
+  private void applyRecommendations(
+      final AutoTuner tuner, final List<TableRecommendation> actionable, final boolean runAnalyze) {
+    List<List<String>> rows = new ArrayList<>();
+    for (TableRecommendation rec : actionable) {
+      rows.add(applyOne(tuner, rec, runAnalyze));
+    }
+    printToAsciiTable(
+        List.of("Table", "Action", "Status", "Details"), rows, "No recommendations applied");
+  }
+
+  private List<String> applyOne(
+      final AutoTuner tuner, final TableRecommendation rec, final boolean runAnalyze) {
+    try {
+      jdbi.useHandle(handle -> tuner.apply(handle, rec));
+      if (runAnalyze) {
+        jdbi.useHandle(handle -> tuner.analyzeOne(handle, rec.tableName()));
+        return List.of(rec.tableName(), rec.action().name(), "OK", "Applied + analyzed");
+      }
+      return List.of(rec.tableName(), rec.action().name(), "OK", "Applied");
+    } catch (Exception e) {
+      LOG.error("Failed to apply recommendation for {}: {}", rec.tableName(), e.getMessage(), e);
+      String detail = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName();
+      return List.of(rec.tableName(), rec.action().name(), "FAILED", detail);
+    }
+  }
+
   /**
    * Unlike most ops commands (e.g. deploy-pipelines) that delegate to the server API, this command
    * operates directly on the database. This is intentional: when JWT signing keys have been rotated,

openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Action.java

@@ -0,0 +1,25 @@
+/*
+ *  Copyright 2026 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.openmetadata.service.util.dbtune;
+
+public enum Action {
+  APPLY,
+  TIGHTEN,
+  RELAX,
+  OK,
+  SKIP;
+
+  public boolean isActionable() {
+    return this == APPLY || this == TIGHTEN || this == RELAX;
+  }
+}