diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DbTuneIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DbTuneIT.java new file mode 100644 index 000000000000..fd7a3978d570 --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DbTuneIT.java @@ -0,0 +1,224 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.it.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Map; +import org.jdbi.v3.core.Jdbi; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.openmetadata.it.bootstrap.TestSuiteBootstrap; +import org.openmetadata.service.jdbi3.locator.ConnectionType; +import org.openmetadata.service.util.dbtune.Action; +import org.openmetadata.service.util.dbtune.AutoTuner; +import org.openmetadata.service.util.dbtune.DbTuneDiagnosis; +import org.openmetadata.service.util.dbtune.DbTuneResult; +import org.openmetadata.service.util.dbtune.Diagnostic; +import org.openmetadata.service.util.dbtune.MysqlAutoTuner; +import org.openmetadata.service.util.dbtune.MysqlDiagnostic; +import org.openmetadata.service.util.dbtune.PostgresAutoTuner; +import org.openmetadata.service.util.dbtune.PostgresDiagnostic; +import org.openmetadata.service.util.dbtune.TableRecommendation; + +/** + * End-to-end tests for {@link AutoTuner} against the live Testcontainers database. + * + *

The read-only tests ({@link #analyzeReturnsRecommendationsForKnownTables}, {@link + * #dryRunDoesNotMutateReloptions}) run against the real catalog tables that the IT bootstrap + * created via migrations. + * + *

Tests that exercise the write path ({@link #applyExecutesAndIsIdempotent}, {@link + * #analyzeOneRunsOnIsolatedTable}) deliberately use a private throwaway table — never a real + * catalog table. Reason: {@code ALTER TABLE} on a shared production table bumps MySQL's per-table + * metadata version, which invalidates JDBC prepared-statement caches across the whole + * Testcontainer. When that table has a {@code JSON} column (e.g. {@code entity_relationship}), the + * driver's re-prepared metadata sometimes returns the column type as {@code VARBINARY}, and + * subsequent {@code INSERT} statements fail with {@code "Cannot create a JSON value from a string + * with CHARACTER SET 'binary'"}. We saw this break {@code GlossaryTermRelationsIT}, + * {@code DomainResourceIT}, and the lineage ITs in CI when an earlier version of this test applied + * settings to {@code entity_relationship}. The recommendations themselves are sound — the IT just + * cannot afford the side effect on a shared DB. + * + *

Sequential because {@code @BeforeEach} / {@code @AfterEach} create and drop the same isolated + * table by name; concurrent execution would race. + */ +@Execution(ExecutionMode.SAME_THREAD) +class DbTuneIT { + + /** Table created and dropped per test — never a catalog table. Safe blast radius. */ + private static final String ISOLATED_TABLE = "dbtune_it_isolated_table"; + + /** A real catalog table used only by the read-only tests to assert against the live schema. */ + private static final String READ_ONLY_PROBE_TABLE = "entity_relationship"; + + @BeforeEach + void createIsolatedTable() { + Jdbi jdbi = TestSuiteBootstrap.getJdbi(); + ConnectionType connType = currentConnectionType(); + jdbi.useHandle( + handle -> { + handle.execute("DROP TABLE IF EXISTS " + quoteIdent(connType, ISOLATED_TABLE)); + if (connType == ConnectionType.POSTGRES) { + handle.execute( + "CREATE TABLE " + quoteIdent(connType, ISOLATED_TABLE) + " (id INT PRIMARY KEY)"); + } else { + handle.execute( + "CREATE TABLE " + + quoteIdent(connType, ISOLATED_TABLE) + + " (id INT PRIMARY KEY) ENGINE=InnoDB"); + } + }); + } + + @AfterEach + void dropIsolatedTable() { + Jdbi jdbi = TestSuiteBootstrap.getJdbi(); + ConnectionType connType = currentConnectionType(); + jdbi.useHandle( + handle -> handle.execute("DROP TABLE IF EXISTS " + quoteIdent(connType, ISOLATED_TABLE))); + } + + @Test + void analyzeReturnsRecommendationsForKnownTables() { + AutoTuner tuner = currentTuner(); + Jdbi jdbi = TestSuiteBootstrap.getJdbi(); + + DbTuneResult result = jdbi.withHandle(tuner::analyze); + + assertNotNull(result); + assertNotNull(result.engineVersion()); + assertFalse(result.tableRecommendations().isEmpty(), "Expected at least one recommendation"); + assertTrue( + result.tableRecommendations().stream() + .anyMatch(r -> READ_ONLY_PROBE_TABLE.equals(r.tableName())), + READ_ONLY_PROBE_TABLE + " should be in the recommendations"); + } + + @Test + void applyExecutesAndIsIdempotent() { + AutoTuner tuner = currentTuner(); + Jdbi jdbi = TestSuiteBootstrap.getJdbi(); + ConnectionType connType = currentConnectionType(); + TableRecommendation rec = recommendationForIsolatedTable(connType); + + jdbi.useHandle(handle -> tuner.apply(handle, rec)); + + String built = tuner.buildAlterStatement(rec); + assertTrue(built.contains(ISOLATED_TABLE), "ALTER target table mismatch: " + built); + + // Apply twice — second invocation must complete without throwing. + jdbi.useHandle(handle -> tuner.apply(handle, rec)); + } + + @Test + void analyzeOneRunsOnIsolatedTable() { + AutoTuner tuner = currentTuner(); + Jdbi jdbi = TestSuiteBootstrap.getJdbi(); + + jdbi.useHandle(handle -> tuner.analyzeOne(handle, ISOLATED_TABLE)); + } + + @Test + void diagnoseCompletesWithoutErrorAndReturnsStructuredResult() { + Diagnostic diagnostic = currentDiagnostic(); + Jdbi jdbi = TestSuiteBootstrap.getJdbi(); + + DbTuneDiagnosis diagnosis = jdbi.withHandle(diagnostic::diagnose); + + assertNotNull(diagnosis, "diagnose() must return a non-null diagnosis"); + assertNotNull(diagnosis.findings(), "findings list must be present (empty allowed)"); + assertNotNull(diagnosis.notes(), "notes list must be present (empty allowed)"); + // On a freshly-bootstrapped IT DB we expect either: + // - an empty diagnosis (nothing has accumulated yet to flag), OR + // - notes about missing optional extensions like pg_stat_statements. + // Either is fine — what we're really asserting is the diagnostic ran end-to-end without + // throwing on the live schema. + } + + @Test + void dryRunDoesNotMutateReloptions() { + AutoTuner tuner = currentTuner(); + Jdbi jdbi = TestSuiteBootstrap.getJdbi(); + + Map before = currentSettingsFor(tuner, jdbi, READ_ONLY_PROBE_TABLE); + + DbTuneResult result = jdbi.withHandle(tuner::analyze); + assertNotNull(result); + + Map after = currentSettingsFor(tuner, jdbi, READ_ONLY_PROBE_TABLE); + assertEquals(before, after, "Analyze (dry-run) must not change table settings"); + } + + // ---- helpers ---- + + private AutoTuner currentTuner() { + return currentConnectionType() == ConnectionType.POSTGRES + ? new PostgresAutoTuner() + : new MysqlAutoTuner(); + } + + private Diagnostic currentDiagnostic() { + return currentConnectionType() == ConnectionType.POSTGRES + ? new PostgresDiagnostic() + : new MysqlDiagnostic(); + } + + private ConnectionType currentConnectionType() { + return "mysql".equalsIgnoreCase(System.getProperty("databaseType", "postgres")) + ? ConnectionType.MYSQL + : ConnectionType.POSTGRES; + } + + /** + * Builds a {@link TableRecommendation} pointing at {@link #ISOLATED_TABLE} with engine-appropriate + * settings. We construct it directly rather than going through {@code analyze()} because the + * isolated table is intentionally NOT in the static catalog — that's how we keep the apply path + * off shared production tables. + */ + private TableRecommendation recommendationForIsolatedTable(final ConnectionType connType) { + Map recommended = + connType == ConnectionType.POSTGRES + ? Map.of("autovacuum_vacuum_scale_factor", "0.05") + : Map.of("STATS_PERSISTENT", "1", "STATS_AUTO_RECALC", "1"); + return new TableRecommendation( + ISOLATED_TABLE, Action.APPLY, 0L, 0L, Map.of(), recommended, "Isolated IT test table"); + } + + /** + * Re-runs analyze and projects out the {@link TableRecommendation#currentSettings()} for the + * named table. Going through the same code path that built the original recommendation keeps the + * assertion stable across either dialect's parsing rules. + */ + private Map currentSettingsFor( + final AutoTuner tuner, final Jdbi jdbi, final String tableName) { + return jdbi.withHandle(tuner::analyze).tableRecommendations().stream() + .filter(r -> tableName.equals(r.tableName())) + .findFirst() + .map(TableRecommendation::currentSettings) + .orElse(Map.of()); + } + + private static String quoteIdent(final ConnectionType connType, final String identifier) { + if (!identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { + throw new IllegalArgumentException("Refusing unsafe identifier: " + identifier); + } + return connType == ConnectionType.POSTGRES ? "\"" + identifier + "\"" : "`" + identifier + "`"; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index 66a8d79c2263..8a28a1c4b198 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -133,6 +133,16 @@ import org.openmetadata.service.secrets.SecretsManagerUpdateService; import org.openmetadata.service.security.auth.SecurityConfigurationManager; import org.openmetadata.service.security.jwt.JWTTokenGenerator; +import org.openmetadata.service.util.dbtune.AutoTuner; +import org.openmetadata.service.util.dbtune.DbTuneDiagnosis; +import org.openmetadata.service.util.dbtune.DbTuneReport; +import org.openmetadata.service.util.dbtune.DbTuneResult; +import org.openmetadata.service.util.dbtune.Diagnostic; +import org.openmetadata.service.util.dbtune.MysqlAutoTuner; +import org.openmetadata.service.util.dbtune.MysqlDiagnostic; +import org.openmetadata.service.util.dbtune.PostgresAutoTuner; +import org.openmetadata.service.util.dbtune.PostgresDiagnostic; +import org.openmetadata.service.util.dbtune.TableRecommendation; import org.openmetadata.service.util.jdbi.DatabaseAuthenticationProviderFactory; import org.openmetadata.service.util.jdbi.JdbiUtils; import org.slf4j.LoggerFactory; @@ -175,9 +185,13 @@ public Integer call() { + "'drop-create', 'changelog', 'migrate', 'migrate-secrets', 'reindex', 'reembed', 'reindex-rdf', 'reindexdi', 'deploy-pipelines', " + "'dbServiceCleanup', 'relationshipCleanup', 'tagUsageCleanup', 'drop-indexes', 'remove-security-config', 'create-indexes', " + "'setOpenMetadataUrl', 'configureEmailSettings', 'get-security-config', 'update-security-config', 'install-app', 'delete-app', 'create-user', 'reset-password', " - + "'syncAlertOffset', 'analyze-tables', 'cleanup-flowable-history', 'regenerate-bot-tokens'"); + + "'syncAlertOffset', 'analyze-tables', 'db-tune', 'cleanup-flowable-history', 'regenerate-bot-tokens'"); LOG.info( "Use 'reindex --auto-tune' for automatic performance optimization based on cluster capabilities"); + LOG.info( + "Use 'db-tune' for a per-table autovacuum / InnoDB stats tuning report; add --apply to " + + "execute the recommendations, --analyze to refresh planner stats on changed tables, " + + "and --diagnose to surface unused indexes, bloat, slow queries, and other DBA findings"); LOG.info( "Use 'cleanup-flowable-history --delete --runtime-batch-size=1000 --history-batch-size=1000' for Flowable cleanup with custom options"); LOG.info( @@ -2469,6 +2483,132 @@ public Integer analyzeTables() { } } + @Command( + name = "db-tune", + description = + "Generate a per-table autovacuum / InnoDB stats tuning report and optionally apply it. " + + "Default mode is read-only — pass --apply to execute the ALTER TABLE statements, " + + "--analyze to refresh planner stats on changed tables, and --diagnose to also " + + "surface unused indexes, bloat, slow queries, and other read-only DBA findings.") + public Integer dbTune( + @Option( + names = {"--apply"}, + defaultValue = "false", + description = + "Apply the recommendations. Without this flag the command only prints the report.") + boolean apply, + @Option( + names = {"--yes", "-y"}, + defaultValue = "false", + description = "Skip the interactive confirmation when applying.") + boolean skipPrompt, + @Option( + names = {"--analyze"}, + defaultValue = "false", + description = + "After --apply, run ANALYZE on each changed table so planner stats reflect the new settings.") + boolean runAnalyze, + @Option( + names = {"--diagnose"}, + defaultValue = "false", + description = + "Also run a read-only diagnostic pass (unused indexes, bloat, low cache hit, " + + "stale ANALYZE, seq-scan-heavy tables, slow queries). Pure inspection — " + + "never modifies anything.") + boolean runDiagnose) { + try { + parseConfig(); + String driverClass = config.getDataSourceFactory().getDriverClass(); + ConnectionType connType = ConnectionType.from(driverClass); + if (connType == null) { + LOG.error( + "db-tune does not support driver class '{}'. Only the bundled MySQL and PostgreSQL drivers are recognised.", + driverClass); + return 1; + } + AutoTuner tuner = autoTunerFor(connType); + DbTuneResult result = jdbi.withHandle(tuner::analyze); + LOG.info("\n{}", DbTuneReport.render(result)); + if (runDiagnose) { + Diagnostic diagnostic = diagnosticFor(connType); + DbTuneDiagnosis diagnosis = jdbi.withHandle(diagnostic::diagnose); + LOG.info("\n{}", DbTuneReport.renderDiagnosis(diagnosis)); + } + if (!apply) { + return 0; + } + List actionable = result.actionableRecommendations(); + if (actionable.isEmpty()) { + if (result.tableRecommendations().isEmpty()) { + LOG.info("Nothing to apply — no tracked tables exist on this database."); + } else { + LOG.info( + "Nothing to apply — every tracked table already matches its recommended settings."); + } + return 0; + } + if (!skipPrompt && !confirmApply(tuner, actionable)) { + LOG.info("Operation cancelled."); + return 0; + } + applyRecommendations(tuner, actionable, runAnalyze); + return 0; + } catch (Exception e) { + LOG.error("db-tune failed due to ", e); + return 1; + } + } + + private AutoTuner autoTunerFor(final ConnectionType connType) { + return switch (connType) { + case POSTGRES -> new PostgresAutoTuner(); + case MYSQL -> new MysqlAutoTuner(); + }; + } + + private Diagnostic diagnosticFor(final ConnectionType connType) { + return switch (connType) { + case POSTGRES -> new PostgresDiagnostic(); + case MYSQL -> new MysqlDiagnostic(); + }; + } + + private boolean confirmApply(final AutoTuner tuner, final List actionable) { + LOG.info("About to apply {} ALTER statements:", actionable.size()); + LOG.info("\n{}", DbTuneReport.renderAlterStatements(tuner, actionable)); + @SuppressWarnings("resource") + Scanner scanner = new Scanner(System.in); + LOG.info("Apply now? [y/N]: "); + String input = scanner.hasNext() ? scanner.next().trim().toLowerCase() : ""; + return input.equals("y") || input.equals("yes"); + } + + private void applyRecommendations( + final AutoTuner tuner, final List actionable, final boolean runAnalyze) { + List> rows = new ArrayList<>(); + for (TableRecommendation rec : actionable) { + rows.add(applyOne(tuner, rec, runAnalyze)); + } + printToAsciiTable( + List.of("Table", "Action", "Status", "Details"), rows, "No recommendations applied"); + } + + private List applyOne( + final AutoTuner tuner, final TableRecommendation rec, final boolean runAnalyze) { + try { + jdbi.useHandle(handle -> tuner.apply(handle, rec)); + if (runAnalyze) { + jdbi.useHandle(handle -> tuner.analyzeOne(handle, rec.tableName())); + return List.of(rec.tableName(), rec.action().name(), "OK", "Applied + analyzed"); + } + return List.of(rec.tableName(), rec.action().name(), "OK", "Applied"); + } catch (Exception e) { + LOG.error("Failed to apply recommendation for {}: {}", rec.tableName(), e.getMessage(), e); + String detail = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + return List.of(rec.tableName(), rec.action().name(), "FAILED", detail); + } + } + /** * Unlike most ops commands (e.g. deploy-pipelines) that delegate to the server API, this command * operates directly on the database. This is intentional: when JWT signing keys have been rotated, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Action.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Action.java new file mode 100644 index 000000000000..199115fe65d8 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Action.java @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +public enum Action { + APPLY, + TIGHTEN, + RELAX, + OK, + SKIP; + + public boolean isActionable() { + return this == APPLY || this == TIGHTEN || this == RELAX; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/AutoTuner.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/AutoTuner.java new file mode 100644 index 000000000000..07f34f6cc3f4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/AutoTuner.java @@ -0,0 +1,50 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import org.jdbi.v3.core.Handle; + +/** + * Engine-specific auto-tuner. Implementations: + * + *

+ */ +public interface AutoTuner { + + /** Read stats + settings, then turn them into recommendations. Mixes I/O and pure logic. */ + DbTuneResult analyze(Handle handle); + + /** + * Pure decision function. Given observed table stats, return the recommendation. Exposed + * separately so unit tests can assert the heuristic without hitting a database. + */ + TableRecommendation recommend(TableStats stats); + + /** + * Apply a single actionable recommendation. No-op for non-actionable actions. Idempotent — safe + * to re-run. + */ + void apply(Handle handle, TableRecommendation recommendation); + + /** Refresh planner stats for one table after a settings change. */ + void analyzeOne(Handle handle, String tableName); + + /** Build the {@code ALTER TABLE} statement for a recommendation. Engine-specific syntax. */ + String buildAlterStatement(TableRecommendation recommendation); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/DbTuneDiagnosis.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/DbTuneDiagnosis.java new file mode 100644 index 000000000000..a2e88ac006a9 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/DbTuneDiagnosis.java @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** Diagnostic result bundle. {@code notes} carries advisory messages (e.g. missing extension). */ +public record DbTuneDiagnosis(List findings, List notes) { + + public DbTuneDiagnosis { + findings = findings == null ? List.of() : List.copyOf(findings); + notes = notes == null ? List.of() : List.copyOf(notes); + } + + /** Group findings by category preserving the enum order so the report sections print stably. */ + public Map> findingsByCategory() { + return findings.stream() + .collect( + Collectors.groupingBy( + Finding::category, + () -> new java.util.EnumMap<>(DiagnosticCategory.class), + Collectors.toList())); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/DbTuneReport.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/DbTuneReport.java new file mode 100644 index 000000000000..439e51959c3a --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/DbTuneReport.java @@ -0,0 +1,203 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; +import org.openmetadata.service.util.AsciiTable; + +public final class DbTuneReport { + + private static final NumberFormat ROW_FORMAT = NumberFormat.getInstance(Locale.ROOT); + private static final long KB = 1024L; + private static final long MB = KB * 1024L; + private static final long GB = MB * 1024L; + + private DbTuneReport() {} + + public static String render(final DbTuneResult result) { + StringBuilder out = new StringBuilder(); + out.append("Database engine: ").append(result.engine()); + if (result.engineVersion() != null && !result.engineVersion().isBlank()) { + out.append(" ").append(result.engineVersion()); + } + out.append('\n').append('\n'); + appendServerParams(out, result.serverParams()); + appendTableRecommendations(out, result.tableRecommendations()); + appendNextSteps( + out, result.tableRecommendations().size(), result.actionableRecommendations().size()); + return out.toString(); + } + + private static void appendServerParams( + final StringBuilder out, final List checks) { + out.append("=== Server-level parameter compliance ===\n"); + if (checks.isEmpty()) { + out.append("(no parameter-group checks for this engine)\n\n"); + return; + } + List headers = List.of("Parameter", "Current", "Recommended", "Status", "Note"); + List> rows = + checks.stream() + .map( + c -> + List.of( + nullToBlank(c.parameter()), + nullToBlank(c.currentValue()), + nullToBlank(c.recommendedValue()), + nullToBlank(c.status()), + nullToBlank(c.note()))) + .toList(); + out.append(new AsciiTable(headers, rows, true, "", "(empty)").render()); + out.append('\n'); + out.append( + "These cannot be applied by this tool — change them in your DB parameter group / RDS console.\n\n"); + } + + private static void appendTableRecommendations( + final StringBuilder out, final List recs) { + out.append("=== Per-table recommendations (").append(recs.size()).append(" tables) ===\n"); + if (recs.isEmpty()) { + out.append("(no recommendations — none of the tracked tables exist on this database)\n\n"); + return; + } + List headers = + List.of("Table", "Rows", "Size", "Current", "Recommended", "Action", "Reason"); + List> rows = + recs.stream() + .map( + r -> + List.of( + r.tableName(), + ROW_FORMAT.format(r.rowCount()), + formatBytes(r.totalBytes()), + formatSettings(r.currentSettings()), + formatSettings(r.recommendedSettings()), + r.action().name(), + nullToBlank(r.reason()))) + .toList(); + out.append(new AsciiTable(headers, rows, true, "", "(empty)").render()); + out.append('\n'); + } + + private static void appendNextSteps( + final StringBuilder out, final int totalRecommendations, final int actionableCount) { + if (totalRecommendations == 0) { + // No tracked tables exist on this database — saying "all match" would be misleading. + return; + } + if (actionableCount == 0) { + out.append("All tracked tables already match their recommended settings — nothing to do.\n"); + return; + } + out.append("Next steps:\n"); + out.append( + " ./bootstrap/openmetadata-ops.sh db-tune --apply --analyze # apply + refresh planner stats\n"); + out.append( + " ./bootstrap/openmetadata-ops.sh db-tune --apply # apply only; run analyze-tables later\n"); + } + + static String formatSettings(final Map settings) { + if (settings == null || settings.isEmpty()) { + return "(default)"; + } + return settings.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .map(e -> e.getKey() + "=" + e.getValue()) + .collect(Collectors.joining(", ")); + } + + static String formatBytes(final long bytes) { + if (bytes <= 0) { + return "0 B"; + } + if (bytes >= GB) { + return String.format(Locale.ROOT, "%.1f GB", bytes / (double) GB); + } + if (bytes >= MB) { + return String.format(Locale.ROOT, "%.0f MB", bytes / (double) MB); + } + if (bytes >= KB) { + return String.format(Locale.ROOT, "%.0f KB", bytes / (double) KB); + } + return bytes + " B"; + } + + private static String nullToBlank(final String value) { + return value == null ? "" : value; + } + + /** Concatenates each recommendation's ALTER statement, one per line, terminated by a semicolon. */ + public static String renderAlterStatements( + final AutoTuner tuner, final List recommendations) { + List lines = new ArrayList<>(recommendations.size()); + for (TableRecommendation rec : recommendations) { + lines.add(tuner.buildAlterStatement(rec) + ";"); + } + return String.join("\n", lines); + } + + /** + * Renders read-only diagnostic findings grouped by category. Each category that produced at + * least one finding gets its own section with a category-specific column layout. Categories with + * zero findings are suppressed; the {@code notes} list is appended at the end so an operator sees + * what couldn't be checked (missing extension, permissions, etc.). + */ + public static String renderDiagnosis(final DbTuneDiagnosis diagnosis) { + StringBuilder out = new StringBuilder(); + out.append("=== Diagnostic findings ===\n"); + Map> grouped = diagnosis.findingsByCategory(); + if (grouped.isEmpty()) { + out.append("(no findings — every check returned a clean result)\n"); + } + for (Map.Entry> e : grouped.entrySet()) { + appendCategorySection(out, e.getKey(), e.getValue()); + } + appendNotes(out, diagnosis.notes()); + return out.toString(); + } + + private static void appendCategorySection( + final StringBuilder out, final DiagnosticCategory category, final List findings) { + out.append('\n') + .append(category.title()) + .append(" (") + .append(findings.size()) + .append(" found):\n"); + out.append(" ").append(category.description()).append('\n'); + List> rows = new ArrayList<>(); + for (Finding f : findings) { + List row = new ArrayList<>(category.columns().size()); + for (String col : category.columns()) { + row.add(nullToBlank(f.attributes().get(col))); + } + rows.add(row); + } + out.append(new AsciiTable(category.columns(), rows, true, "", "(empty)").render()); + out.append('\n'); + } + + private static void appendNotes(final StringBuilder out, final List notes) { + if (notes == null || notes.isEmpty()) { + return; + } + out.append("\nNotes:\n"); + for (String note : notes) { + out.append(" - ").append(note).append('\n'); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/DbTuneResult.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/DbTuneResult.java new file mode 100644 index 000000000000..ead95f324ae8 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/DbTuneResult.java @@ -0,0 +1,32 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.util.List; + +public record DbTuneResult( + String engine, + String engineVersion, + List serverParams, + List tableRecommendations) { + + public DbTuneResult { + serverParams = serverParams == null ? List.of() : List.copyOf(serverParams); + tableRecommendations = + tableRecommendations == null ? List.of() : List.copyOf(tableRecommendations); + } + + public List actionableRecommendations() { + return tableRecommendations.stream().filter(r -> r.action().isActionable()).toList(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Diagnostic.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Diagnostic.java new file mode 100644 index 000000000000..9e10f2af851d --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Diagnostic.java @@ -0,0 +1,26 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import org.jdbi.v3.core.Handle; + +/** + * Read-only DBA diagnostic. Inspects the live database for unused indexes, bloat indicators, slow + * queries, and other signals. Implementations must catch and log per-category errors so a missing + * extension (e.g. {@code pg_stat_statements} not installed) does not abort the whole diagnose run + * — surface it in {@link DbTuneDiagnosis#notes()} instead. + */ +public interface Diagnostic { + + DbTuneDiagnosis diagnose(Handle handle); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/DiagnosticCategory.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/DiagnosticCategory.java new file mode 100644 index 000000000000..c7bc9d2621f4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/DiagnosticCategory.java @@ -0,0 +1,77 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.util.List; + +/** + * Categories of read-only diagnostic findings emitted by {@link Diagnostic#diagnose}. Each category + * has a fixed list of attribute keys that {@link Finding#attributes} is expected to populate; the + * report renderer dispatches column layout per category. + */ +public enum DiagnosticCategory { + UNUSED_INDEX( + "Unused indexes", + "Indexes with zero scans since last stats reset; candidates for DROP after a usage review.", + List.of("table", "index", "size", "scans")), + HIGH_DEAD_TUPLES( + "Tables with high dead-tuple ratio", + "n_dead_tup / n_live_tup > 0.2 — autovacuum is falling behind on this table.", + List.of("table", "live_rows", "dead_rows", "dead_ratio", "last_vacuum")), + LOW_CACHE_HIT( + "Tables with low cache hit ratio", + "Heap reads exceed 1000 with hit ratio < 90%; suggests undersized buffers or hot seq scans.", + List.of("table", "heap_reads", "heap_hits", "hit_pct")), + STALE_STATS( + "Tables with stale ANALYZE", + "Last autoanalyze older than 14 days (or never); planner stats may be misleading.", + List.of("table", "last_analyzed", "live_rows")), + SEQ_SCAN_HEAVY( + "Tables with seq-scan-heavy access", + "seq_scan/idx_scan > 10 with > 1000 seq scans; suggests a missing index.", + List.of("table", "seq_scans", "idx_scans", "ratio")), + SLOW_QUERY( + "Top slowest queries", + "From pg_stat_statements / events_statements_summary_by_digest. Truncated to 100 chars.", + List.of("query", "calls", "mean_ms")), + FULL_TABLE_SCAN( + "Queries doing full table scans", + "From sys.statements_with_full_table_scans (MySQL).", + List.of("query", "exec_count", "rows_examined_avg")), + LOW_BUFFER_POOL_HIT( + "InnoDB buffer pool hit ratio", + "Hit ratio < 99% suggests undersized innodb_buffer_pool_size for the working set.", + List.of("metric", "value")); + + private final String title; + private final String description; + private final List columns; + + DiagnosticCategory(final String title, final String description, final List columns) { + this.title = title; + this.description = description; + this.columns = List.copyOf(columns); + } + + public String title() { + return title; + } + + public String description() { + return description; + } + + public List columns() { + return columns; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Finding.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Finding.java new file mode 100644 index 000000000000..7026f9df5430 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Finding.java @@ -0,0 +1,28 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.util.Map; + +/** + * One row of a diagnostic finding. {@code attributes} keys must match {@link + * DiagnosticCategory#columns()} for the same {@code category} so the renderer can lay them out + * predictably. + */ +public record Finding( + DiagnosticCategory category, Severity severity, Map attributes) { + + public Finding { + attributes = attributes == null ? Map.of() : Map.copyOf(attributes); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/MysqlAutoTuner.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/MysqlAutoTuner.java new file mode 100644 index 000000000000..66d785be9bd9 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/MysqlAutoTuner.java @@ -0,0 +1,251 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; +import org.jdbi.v3.core.Handle; +import org.openmetadata.service.util.dbtune.MysqlTuningCatalog.Profile; + +public final class MysqlAutoTuner implements AutoTuner { + + @Override + public DbTuneResult analyze(final Handle handle) { + String version = readVersion(handle); + List serverParams = readServerParams(handle); + List stats = loadTableStats(handle); + List recs = stats.stream().map(this::recommend).toList(); + return new DbTuneResult("MySQL", version, serverParams, recs); + } + + @Override + public TableRecommendation recommend(final TableStats stats) { + Profile profile = MysqlTuningCatalog.profileFor(stats.tableName()); + if (profile == null) { + return skip(stats, "Table is not in the dbtune catalog"); + } + if (stats.rowCount() < profile.rowThreshold()) { + return skip( + stats, + String.format( + Locale.ROOT, + "Row count %d below threshold %d", + stats.rowCount(), + profile.rowThreshold())); + } + return decideAction(stats, profile); + } + + private TableRecommendation decideAction(final TableStats stats, final Profile profile) { + Map recommended = profile.settings(); + Map current = stats.currentSettings(); + if (settingsMatch(current, recommended)) { + return new TableRecommendation( + stats.tableName(), + Action.OK, + stats.rowCount(), + stats.totalBytes(), + current, + recommended, + "Already matches recommended settings"); + } + Action action = current.isEmpty() ? Action.APPLY : Action.TIGHTEN; + return new TableRecommendation( + stats.tableName(), + action, + stats.rowCount(), + stats.totalBytes(), + current, + recommended, + profile.reason()); + } + + @Override + public void apply(final Handle handle, final TableRecommendation recommendation) { + if (!recommendation.action().isActionable()) { + return; + } + handle.execute(buildAlterStatement(recommendation)); + } + + @Override + public void analyzeOne(final Handle handle, final String tableName) { + handle.execute("ANALYZE TABLE " + quoteIdent(tableName)); + } + + @Override + public String buildAlterStatement(final TableRecommendation recommendation) { + String settings = + recommendation.recommendedSettings().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .map(e -> e.getKey() + "=" + e.getValue()) + .collect(Collectors.joining(", ")); + return "ALTER TABLE " + quoteIdent(recommendation.tableName()) + " " + settings; + } + + // ---- DB I/O ---- + + String readVersion(final Handle handle) { + return handle.createQuery("SELECT VERSION()").mapTo(String.class).findOne().orElse(""); + } + + List loadTableStats(final Handle handle) { + List result = new ArrayList<>(); + for (String tableName : MysqlTuningCatalog.tableNames()) { + TableStats stats = loadTableStats(handle, tableName); + if (stats != null) { + result.add(stats); + } + } + return result; + } + + TableStats loadTableStats(final Handle handle, final String tableName) { + return handle + .createQuery( + "SELECT TABLE_ROWS AS rows_estimate, " + + " COALESCE(DATA_LENGTH, 0) AS heap_bytes, " + + " COALESCE(INDEX_LENGTH, 0) AS idx_bytes, " + + " COALESCE(CREATE_OPTIONS, '') AS create_opts " + + "FROM information_schema.TABLES " + + "WHERE TABLE_SCHEMA = DATABASE() " + + " AND TABLE_NAME = :name") + .bind("name", tableName) + .map( + (rs, ctx) -> + new TableStats( + tableName, + Math.max(rs.getLong("rows_estimate"), 0), + rs.getLong("heap_bytes"), + rs.getLong("idx_bytes"), + parseCreateOptions(rs.getString("create_opts")))) + .findOne() + .orElse(null); + } + + List readServerParams(final Handle handle) { + List checks = new ArrayList<>(); + Map recommendations = recommendedServerParams(); + for (Map.Entry e : recommendations.entrySet()) { + String name = e.getKey(); + String recommended = e.getValue(); + String current = readGlobalVariable(handle, name); + checks.add(buildServerCheck(name, current, recommended)); + } + return checks; + } + + // ---- helpers ---- + + static Map parseCreateOptions(final String createOptions) { + if (createOptions == null || createOptions.isBlank()) { + return Map.of(); + } + Map out = new LinkedHashMap<>(); + for (String token : createOptions.trim().split("\\s+")) { + int eq = token.indexOf('='); + if (eq > 0) { + String key = token.substring(0, eq).toUpperCase(Locale.ROOT); + String value = token.substring(eq + 1); + if (key.startsWith("STATS_")) { + out.put(key, value); + } + } + } + return Map.copyOf(out); + } + + static boolean settingsMatch(final Map current, final Map rec) { + for (Map.Entry e : rec.entrySet()) { + String currentValue = current.get(e.getKey()); + if (currentValue == null || !numericEquals(currentValue, e.getValue())) { + return false; + } + } + return true; + } + + private static boolean numericEquals(final String a, final String b) { + try { + return Double.parseDouble(a) == Double.parseDouble(b); + } catch (NumberFormatException ex) { + return a.equalsIgnoreCase(b); + } + } + + private static TableRecommendation skip(final TableStats stats, final String reason) { + return new TableRecommendation( + stats.tableName(), + Action.SKIP, + stats.rowCount(), + stats.totalBytes(), + stats.currentSettings(), + Map.of(), + reason); + } + + static String quoteIdent(final String identifier) { + if (!identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { + throw new IllegalArgumentException( + "Refusing to build SQL with unsafe identifier: " + identifier); + } + return "`" + identifier + "`"; + } + + private String readGlobalVariable(final Handle handle, final String name) { + return handle + .createQuery( + "SELECT VARIABLE_VALUE FROM performance_schema.global_variables " + + "WHERE VARIABLE_NAME = :n") + .bind("n", name.toLowerCase(Locale.ROOT)) + .mapTo(String.class) + .findOne() + .orElse(null); + } + + static Map recommendedServerParams() { + Map map = new LinkedHashMap<>(); + map.put("innodb_buffer_pool_size", "40-60% of RAM (use formula form on RDS)"); + map.put("innodb_io_capacity", "2000"); + map.put("innodb_io_capacity_max", "4000"); + map.put("innodb_stats_persistent_sample_pages", "64"); + map.put("sort_buffer_size", "8388608"); // 8 MB + map.put("join_buffer_size", "4194304"); // 4 MB + map.put("tmp_table_size", "67108864"); // 64 MB + map.put("max_heap_table_size", "67108864"); // 64 MB + return Map.copyOf(map); + } + + static ServerParamCheck buildServerCheck( + final String name, final String current, final String recommended) { + if (current == null) { + return new ServerParamCheck( + name, "", recommended, ServerParamCheck.STATUS_UNKNOWN, "Variable not visible"); + } + if (recommended.contains("%")) { + return new ServerParamCheck( + name, + current, + recommended, + ServerParamCheck.STATUS_UNTUNED, + "RAM-relative; verify in RDS"); + } + boolean ok = numericEquals(current, recommended); + String status = ok ? ServerParamCheck.STATUS_OK : ServerParamCheck.STATUS_MISMATCH; + return new ServerParamCheck(name, current, recommended, status, ""); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/MysqlDiagnostic.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/MysqlDiagnostic.java new file mode 100644 index 000000000000..e957be7db2e4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/MysqlDiagnostic.java @@ -0,0 +1,189 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.core.Handle; + +/** + * MySQL diagnostic. Reads from {@code sys.*}, {@code performance_schema.*}, and + * {@code INFORMATION_SCHEMA} views; gracefully degrades if a view is missing or permissions are + * insufficient (the operator gets a {@link DbTuneDiagnosis#notes()} entry). + */ +@Slf4j +public final class MysqlDiagnostic implements Diagnostic { + + static final double LOW_BUFFER_POOL_HIT = 0.99; + static final int SLOW_QUERY_LIMIT = 10; + static final int QUERY_TRUNCATE = 100; + + @Override + public DbTuneDiagnosis diagnose(final Handle handle) { + List findings = new ArrayList<>(); + List notes = new ArrayList<>(); + runCategory(handle, notes, "unused indexes", h -> findings.addAll(unusedIndexes(h))); + runCategory(handle, notes, "buffer pool hit", h -> findings.addAll(bufferPoolHit(h, notes))); + runCategory(handle, notes, "slow queries", h -> findings.addAll(slowQueries(h, notes))); + runCategory(handle, notes, "full table scans", h -> findings.addAll(fullTableScans(h, notes))); + return new DbTuneDiagnosis(findings, notes); + } + + private void runCategory( + final Handle handle, + final List notes, + final String label, + final java.util.function.Consumer body) { + try { + body.accept(handle); + } catch (Exception e) { + LOG.warn("Diagnostic [{}] failed: {}", label, e.getMessage()); + notes.add(label + ": " + e.getMessage()); + } + } + + // ---- categories ---- + + List unusedIndexes(final Handle handle) { + return handle + .createQuery( + "SELECT object_schema, object_name, index_name " + + "FROM sys.schema_unused_indexes " + + "WHERE object_schema = DATABASE() " + + "ORDER BY object_name " + + "LIMIT 50") + .map( + (rs, ctx) -> + new Finding( + DiagnosticCategory.UNUSED_INDEX, + Severity.WARN, + Map.of( + "table", + rs.getString("object_name"), + "index", + rs.getString("index_name"), + "size", + "(not in view)", + "scans", + "0"))) + .list(); + } + + List bufferPoolHit(final Handle handle, final List notes) { + Long reads = readGlobalStatusLong(handle, "Innodb_buffer_pool_reads"); + Long requests = readGlobalStatusLong(handle, "Innodb_buffer_pool_read_requests"); + if (reads == null || requests == null || requests == 0) { + notes.add("buffer pool hit: Innodb_buffer_pool_* counters not available"); + return List.of(); + } + double hitRatio = 1.0 - (reads.doubleValue() / requests.doubleValue()); + if (hitRatio >= LOW_BUFFER_POOL_HIT) { + return List.of(); + } + return List.of( + new Finding( + DiagnosticCategory.LOW_BUFFER_POOL_HIT, + Severity.INFO, + Map.of( + "metric", + "innodb_buffer_pool_hit_ratio", + "value", + String.format(Locale.ROOT, "%.4f", hitRatio)))); + } + + List slowQueries(final Handle handle, final List notes) { + try { + return handle + .createQuery( + "SELECT digest_text, count_star AS calls, " + + " ROUND(avg_timer_wait/1000000, 2) AS mean_us " + + "FROM performance_schema.events_statements_summary_by_digest " + + "WHERE schema_name = DATABASE() " + + " AND digest_text IS NOT NULL " + + "ORDER BY avg_timer_wait DESC " + + "LIMIT :limit") + .bind("limit", SLOW_QUERY_LIMIT) + .map( + (rs, ctx) -> { + Map attrs = new LinkedHashMap<>(); + attrs.put("query", truncate(rs.getString("digest_text"))); + attrs.put("calls", String.valueOf(rs.getLong("calls"))); + attrs.put( + "mean_ms", + String.format(Locale.ROOT, "%.2f", rs.getDouble("mean_us") / 1000.0)); + return new Finding(DiagnosticCategory.SLOW_QUERY, Severity.INFO, attrs); + }) + .list(); + } catch (Exception e) { + notes.add("slow queries: performance_schema not available (" + e.getMessage() + ")"); + return List.of(); + } + } + + List fullTableScans(final Handle handle, final List notes) { + try { + return handle + .createQuery( + "SELECT query, exec_count, rows_examined_avg " + + "FROM sys.statements_with_full_table_scans " + + "WHERE db = DATABASE() " + + "ORDER BY exec_count DESC " + + "LIMIT 10") + .map( + (rs, ctx) -> + new Finding( + DiagnosticCategory.FULL_TABLE_SCAN, + Severity.INFO, + Map.of( + "query", truncate(rs.getString("query")), + "exec_count", String.valueOf(rs.getLong("exec_count")), + "rows_examined_avg", String.valueOf(rs.getLong("rows_examined_avg"))))) + .list(); + } catch (Exception e) { + notes.add( + "full table scans: sys.statements_with_full_table_scans not available (" + + e.getMessage() + + ")"); + return List.of(); + } + } + + private Long readGlobalStatusLong(final Handle handle, final String name) { + try { + return handle + .createQuery( + "SELECT VARIABLE_VALUE FROM performance_schema.global_status " + + "WHERE VARIABLE_NAME = :n") + .bind("n", name) + .mapTo(Long.class) + .findOne() + .orElse(null); + } catch (Exception e) { + return null; + } + } + + static String truncate(final String query) { + if (query == null) { + return ""; + } + String collapsed = query.replaceAll("\\s+", " ").trim(); + return collapsed.length() <= QUERY_TRUNCATE + ? collapsed + : collapsed.substring(0, QUERY_TRUNCATE) + "…"; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/MysqlTuningCatalog.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/MysqlTuningCatalog.java new file mode 100644 index 000000000000..8bac39f1e13c --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/MysqlTuningCatalog.java @@ -0,0 +1,130 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +/** + * Static catalog of which tables get which MySQL/InnoDB persistent-stats reloptions, and the + * row-count threshold below which we skip tuning that table. + * + *

InnoDB does not expose autovacuum knobs at the per-table level — purge is global. The lever + * that DOES help on large hot tables is bumping {@code STATS_SAMPLE_PAGES} above the default 20 so + * the planner picks the right index against multi-GB JSONB heaps. {@code STATS_PERSISTENT=1} + + * {@code STATS_AUTO_RECALC=1} are the modern InnoDB defaults; we assert them explicitly so a + * tenant with stale my.cnf overrides converges. + */ +final class MysqlTuningCatalog { + + static final String STATS_PERSISTENT = "STATS_PERSISTENT"; + static final String STATS_AUTO_RECALC = "STATS_AUTO_RECALC"; + static final String STATS_SAMPLE_PAGES = "STATS_SAMPLE_PAGES"; + + record Profile(Map settings, long rowThreshold, boolean relax, String reason) { + Profile { + settings = Map.copyOf(settings); + } + } + + private static final Map HOT = + Map.of( + STATS_PERSISTENT, "1", + STATS_AUTO_RECALC, "1", + STATS_SAMPLE_PAGES, "100"); + + private static final Map ENTITY_LARGE = + Map.of( + STATS_PERSISTENT, "1", + STATS_AUTO_RECALC, "1", + STATS_SAMPLE_PAGES, "64"); + + private static final Map ENTITY_SERVICE = + Map.of( + STATS_PERSISTENT, "1", + STATS_AUTO_RECALC, "1", + STATS_SAMPLE_PAGES, "32"); + + private static final long ROW_THRESHOLD_HOT = 0; + private static final long ROW_THRESHOLD_ENTITY_LARGE = 10_000; + private static final long ROW_THRESHOLD_ENTITY_SERVICE = 5_000; + + private static final Map CATALOG = buildCatalog(); + + private MysqlTuningCatalog() {} + + static Map catalog() { + return CATALOG; + } + + static Set tableNames() { + return CATALOG.keySet(); + } + + static Profile profileFor(final String tableName) { + return CATALOG.get(tableName); + } + + private static Map buildCatalog() { + Map map = new LinkedHashMap<>(); + map.put( + "entity_relationship", + new Profile(HOT, ROW_THRESHOLD_HOT, false, "Join target; raise sampling for planner")); + map.put("tag_usage", new Profile(HOT, ROW_THRESHOLD_HOT, false, "Hottest table on read path")); + addEntityLarge(map); + addEntityService(map); + return Map.copyOf(map); + } + + private static void addEntityLarge(final Map map) { + String reason = "Large entity table; bump InnoDB stats sampling"; + for (String t : + new String[] { + "storage_container_entity", + "table_entity", + "dashboard_entity", + "pipeline_entity", + "chart_entity", + "topic_entity", + "ml_model_entity", + "glossary_term_entity", + "metric_entity", + "report_entity", + "search_index_entity", + "api_collection_entity", + "api_endpoint_entity", + "dashboard_data_model_entity", + "ingestion_pipeline_entity", + "data_contract_entity", + "stored_procedure_entity", + "directory_entity", + "file_entity", + "spreadsheet_entity", + "worksheet_entity", + "query_entity" + }) { + map.put(t, new Profile(ENTITY_LARGE, ROW_THRESHOLD_ENTITY_LARGE, false, reason)); + } + } + + private static void addEntityService(final Map map) { + String reason = "Service-tier table; mild stats sampling bump"; + map.put( + "database_entity", + new Profile(ENTITY_SERVICE, ROW_THRESHOLD_ENTITY_SERVICE, false, reason)); + map.put( + "database_schema_entity", + new Profile(ENTITY_SERVICE, ROW_THRESHOLD_ENTITY_SERVICE, false, reason)); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/PostgresAutoTuner.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/PostgresAutoTuner.java new file mode 100644 index 000000000000..3dca35861a9d --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/PostgresAutoTuner.java @@ -0,0 +1,265 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; +import org.jdbi.v3.core.Handle; +import org.openmetadata.service.util.dbtune.PostgresTuningCatalog.Profile; + +public final class PostgresAutoTuner implements AutoTuner { + + private static final List RELOPTION_KEYS = + List.of( + PostgresTuningCatalog.AUTOVACUUM_VACUUM_SCALE_FACTOR, + PostgresTuningCatalog.AUTOVACUUM_ANALYZE_SCALE_FACTOR, + PostgresTuningCatalog.AUTOVACUUM_VACUUM_COST_LIMIT, + PostgresTuningCatalog.AUTOVACUUM_VACUUM_COST_DELAY); + + @Override + public DbTuneResult analyze(final Handle handle) { + String version = readVersion(handle); + List serverParams = readServerParams(handle); + List stats = loadTableStats(handle); + List recs = stats.stream().map(this::recommend).toList(); + return new DbTuneResult("PostgreSQL", version, serverParams, recs); + } + + @Override + public TableRecommendation recommend(final TableStats stats) { + Profile profile = PostgresTuningCatalog.profileFor(stats.tableName()); + if (profile == null) { + return skip(stats, "Table is not in the dbtune catalog"); + } + if (stats.rowCount() < profile.rowThreshold()) { + return skip( + stats, + String.format( + Locale.ROOT, + "Row count %d below threshold %d", + stats.rowCount(), + profile.rowThreshold())); + } + return decideAction(stats, profile); + } + + private TableRecommendation decideAction(final TableStats stats, final Profile profile) { + Map recommended = profile.settings(); + Map current = stats.currentSettings(); + if (settingsMatch(current, recommended)) { + return new TableRecommendation( + stats.tableName(), + Action.OK, + stats.rowCount(), + stats.totalBytes(), + current, + recommended, + "Already matches recommended settings"); + } + Action action = chooseAction(current, profile); + return new TableRecommendation( + stats.tableName(), + action, + stats.rowCount(), + stats.totalBytes(), + current, + recommended, + profile.reason()); + } + + private Action chooseAction(final Map current, final Profile profile) { + if (current.isEmpty()) { + return profile.relax() ? Action.RELAX : Action.APPLY; + } + return profile.relax() ? Action.RELAX : Action.TIGHTEN; + } + + @Override + public void apply(final Handle handle, final TableRecommendation recommendation) { + if (!recommendation.action().isActionable()) { + return; + } + handle.execute(buildAlterStatement(recommendation)); + } + + @Override + public void analyzeOne(final Handle handle, final String tableName) { + handle.execute("ANALYZE " + quoteIdent(tableName)); + } + + @Override + public String buildAlterStatement(final TableRecommendation recommendation) { + String settings = + recommendation.recommendedSettings().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .map(e -> e.getKey() + " = " + e.getValue()) + .collect(Collectors.joining(", ")); + return "ALTER TABLE " + quoteIdent(recommendation.tableName()) + " SET (" + settings + ")"; + } + + // ---- DB I/O ---- + + String readVersion(final Handle handle) { + return handle.createQuery("SHOW server_version").mapTo(String.class).findOne().orElse(""); + } + + List loadTableStats(final Handle handle) { + List result = new ArrayList<>(); + for (String tableName : PostgresTuningCatalog.tableNames()) { + TableStats stats = loadTableStats(handle, tableName); + if (stats != null) { + result.add(stats); + } + } + return result; + } + + TableStats loadTableStats(final Handle handle, final String tableName) { + return handle + .createQuery( + "SELECT c.reltuples::bigint AS rows, " + + " pg_relation_size(c.oid) AS heap_bytes, " + + " pg_indexes_size(c.oid) AS idx_bytes, " + + " COALESCE(c.reloptions, ARRAY[]::text[]) AS opts " + + "FROM pg_class c " + + "JOIN pg_namespace n ON n.oid = c.relnamespace " + + "WHERE c.relkind = 'r' " + + " AND n.nspname = ANY (current_schemas(false)) " + + " AND c.relname = :name") + .bind("name", tableName) + .map( + (rs, ctx) -> { + long rows = rs.getLong("rows"); + long heap = rs.getLong("heap_bytes"); + long idx = rs.getLong("idx_bytes"); + String[] opts = (String[]) rs.getArray("opts").getArray(); + return new TableStats(tableName, Math.max(rows, 0), heap, idx, parseReloptions(opts)); + }) + .findOne() + .orElse(null); + } + + List readServerParams(final Handle handle) { + List checks = new ArrayList<>(); + Map recommendations = recommendedServerParams(); + for (Map.Entry e : recommendations.entrySet()) { + String name = e.getKey(); + String recommended = e.getValue(); + String current = + handle + .createQuery("SELECT setting FROM pg_settings WHERE name = :n") + .bind("n", name) + .mapTo(String.class) + .findOne() + .orElse(null); + checks.add(buildServerCheck(name, current, recommended)); + } + return checks; + } + + // ---- helpers ---- + + static Map parseReloptions(final String[] opts) { + if (opts == null || opts.length == 0) { + return Map.of(); + } + Map out = new LinkedHashMap<>(); + for (String opt : opts) { + int eq = opt.indexOf('='); + if (eq > 0) { + String key = opt.substring(0, eq).toLowerCase(Locale.ROOT); + String value = opt.substring(eq + 1); + if (RELOPTION_KEYS.contains(key)) { + out.put(key, value); + } + } + } + return Map.copyOf(out); + } + + static boolean settingsMatch(final Map current, final Map rec) { + for (Map.Entry e : rec.entrySet()) { + String currentValue = current.get(e.getKey()); + if (currentValue == null || !numericEquals(currentValue, e.getValue())) { + return false; + } + } + return true; + } + + private static boolean numericEquals(final String a, final String b) { + try { + return Double.parseDouble(a) == Double.parseDouble(b); + } catch (NumberFormatException ex) { + return a.equals(b); + } + } + + private static TableRecommendation skip(final TableStats stats, final String reason) { + return new TableRecommendation( + stats.tableName(), + Action.SKIP, + stats.rowCount(), + stats.totalBytes(), + stats.currentSettings(), + Map.of(), + reason); + } + + static String quoteIdent(final String identifier) { + if (!identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { + throw new IllegalArgumentException( + "Refusing to build SQL with unsafe identifier: " + identifier); + } + return "\"" + identifier + "\""; + } + + /** Server-level recommendations from the production runbook. */ + static Map recommendedServerParams() { + Map map = new LinkedHashMap<>(); + map.put("shared_buffers", "40% of RAM (use formula form on RDS)"); + map.put("effective_cache_size", "75% of RAM (use formula form on RDS)"); + map.put("work_mem", "131072"); // 128 MB + map.put("maintenance_work_mem", "2097152"); // 2 GB + map.put("random_page_cost", "1.1"); + map.put("effective_io_concurrency", "200"); + map.put("max_parallel_workers_per_gather", "4"); + map.put("autovacuum_naptime", "15"); + map.put("autovacuum_vacuum_scale_factor", "0.05"); + map.put("autovacuum_analyze_scale_factor", "0.02"); + return Map.copyOf(map); + } + + static ServerParamCheck buildServerCheck( + final String name, final String current, final String recommended) { + if (current == null) { + return new ServerParamCheck( + name, "", recommended, ServerParamCheck.STATUS_UNKNOWN, "Parameter not visible"); + } + if (recommended.contains("%")) { + return new ServerParamCheck( + name, + current, + recommended, + ServerParamCheck.STATUS_UNTUNED, + "RAM-relative; verify in RDS"); + } + boolean ok = numericEquals(current, recommended); + String status = ok ? ServerParamCheck.STATUS_OK : ServerParamCheck.STATUS_MISMATCH; + return new ServerParamCheck(name, current, recommended, status, ""); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/PostgresDiagnostic.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/PostgresDiagnostic.java new file mode 100644 index 000000000000..b2a0991c1268 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/PostgresDiagnostic.java @@ -0,0 +1,253 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.core.Handle; + +/** + * Postgres diagnostic. Each finding category is queried in its own try block so that a missing + * extension or a stat view permission issue surfaces as a {@link DbTuneDiagnosis#notes()} entry + * rather than aborting the whole run. + * + *

Thresholds are baked in for v1; if operators want them tunable later they become CLI flags. + */ +@Slf4j +public final class PostgresDiagnostic implements Diagnostic { + + static final long UNUSED_INDEX_SIZE_BYTES = 10L * 1024 * 1024; + static final double DEAD_TUPLE_RATIO = 0.2; + static final long DEAD_TUPLE_MIN_LIVE_ROWS = 10_000; + static final double LOW_CACHE_HIT_RATIO = 0.9; + static final long LOW_CACHE_HIT_MIN_READS = 1_000; + static final int STALE_STATS_DAYS = 14; + static final long STALE_STATS_MIN_LIVE_ROWS = 1_000; + static final long SEQ_SCAN_RATIO = 10; + static final long SEQ_SCAN_MIN = 1_000; + static final int SLOW_QUERY_LIMIT = 10; + static final long SLOW_QUERY_MIN_CALLS = 100; + static final int QUERY_TRUNCATE = 100; + + @Override + public DbTuneDiagnosis diagnose(final Handle handle) { + List findings = new ArrayList<>(); + List notes = new ArrayList<>(); + runCategory(handle, notes, "unused indexes", h -> findings.addAll(unusedIndexes(h))); + runCategory(handle, notes, "dead tuples", h -> findings.addAll(highDeadTuples(h))); + runCategory(handle, notes, "cache hit", h -> findings.addAll(lowCacheHit(h))); + runCategory(handle, notes, "stale stats", h -> findings.addAll(staleStats(h))); + runCategory(handle, notes, "seq scans", h -> findings.addAll(seqScanHeavy(h))); + runCategory(handle, notes, "slow queries", h -> findings.addAll(slowQueries(h, notes))); + return new DbTuneDiagnosis(findings, notes); + } + + private void runCategory( + final Handle handle, + final List notes, + final String label, + final java.util.function.Consumer body) { + try { + body.accept(handle); + } catch (Exception e) { + LOG.warn("Diagnostic [{}] failed: {}", label, e.getMessage()); + notes.add(label + ": " + e.getMessage()); + } + } + + // ---- categories ---- + + List unusedIndexes(final Handle handle) { + return handle + .createQuery( + "SELECT s.schemaname, s.relname AS table_name, s.indexrelname AS index_name, " + + " s.idx_scan AS scans, " + + " pg_relation_size(s.indexrelid) AS bytes " + + "FROM pg_stat_user_indexes s " + + "JOIN pg_index i ON i.indexrelid = s.indexrelid " + + "WHERE s.idx_scan = 0 " + + " AND NOT i.indisunique " + + " AND NOT i.indisprimary " + + " AND pg_relation_size(s.indexrelid) > :min_bytes " + + "ORDER BY pg_relation_size(s.indexrelid) DESC " + + "LIMIT 50") + .bind("min_bytes", UNUSED_INDEX_SIZE_BYTES) + .map( + (rs, ctx) -> + new Finding( + DiagnosticCategory.UNUSED_INDEX, + Severity.WARN, + Map.of( + "table", rs.getString("table_name"), + "index", rs.getString("index_name"), + "size", DbTuneReport.formatBytes(rs.getLong("bytes")), + "scans", String.valueOf(rs.getLong("scans"))))) + .list(); + } + + List highDeadTuples(final Handle handle) { + return handle + .createQuery( + "SELECT relname AS table_name, " + + " n_live_tup, " + + " n_dead_tup, " + + " ROUND((n_dead_tup::numeric / GREATEST(n_live_tup, 1)) * 100, 2) AS dead_pct, " + + " last_autovacuum " + + "FROM pg_stat_user_tables " + + "WHERE n_live_tup > :min_live " + + " AND n_dead_tup::numeric / GREATEST(n_live_tup, 1) > :threshold " + + "ORDER BY n_dead_tup DESC " + + "LIMIT 25") + .bind("min_live", DEAD_TUPLE_MIN_LIVE_ROWS) + .bind("threshold", DEAD_TUPLE_RATIO) + .map( + (rs, ctx) -> + new Finding( + DiagnosticCategory.HIGH_DEAD_TUPLES, + Severity.WARN, + Map.of( + "table", rs.getString("table_name"), + "live_rows", String.valueOf(rs.getLong("n_live_tup")), + "dead_rows", String.valueOf(rs.getLong("n_dead_tup")), + "dead_ratio", rs.getString("dead_pct") + "%", + "last_vacuum", String.valueOf(rs.getString("last_autovacuum"))))) + .list(); + } + + List lowCacheHit(final Handle handle) { + return handle + .createQuery( + "SELECT relname AS table_name, " + + " heap_blks_read, " + + " heap_blks_hit, " + + " ROUND(heap_blks_hit::numeric / NULLIF(heap_blks_hit + heap_blks_read, 0) * 100, 2) AS hit_pct " + + "FROM pg_statio_user_tables " + + "WHERE heap_blks_read > :min_reads " + + " AND heap_blks_hit::numeric / NULLIF(heap_blks_hit + heap_blks_read, 0) < :threshold " + + "ORDER BY heap_blks_read DESC " + + "LIMIT 25") + .bind("min_reads", LOW_CACHE_HIT_MIN_READS) + .bind("threshold", LOW_CACHE_HIT_RATIO) + .map( + (rs, ctx) -> + new Finding( + DiagnosticCategory.LOW_CACHE_HIT, + Severity.INFO, + Map.of( + "table", rs.getString("table_name"), + "heap_reads", String.valueOf(rs.getLong("heap_blks_read")), + "heap_hits", String.valueOf(rs.getLong("heap_blks_hit")), + "hit_pct", rs.getString("hit_pct") + "%"))) + .list(); + } + + List staleStats(final Handle handle) { + return handle + .createQuery( + "SELECT relname AS table_name, " + + " n_live_tup, " + + " COALESCE(last_autoanalyze, last_analyze) AS last_analyzed " + + "FROM pg_stat_user_tables " + + "WHERE n_live_tup > :min_live " + + " AND (COALESCE(last_autoanalyze, last_analyze) IS NULL " + + " OR COALESCE(last_autoanalyze, last_analyze) < now() - (:days || ' days')::interval) " + + "ORDER BY n_live_tup DESC " + + "LIMIT 25") + .bind("min_live", STALE_STATS_MIN_LIVE_ROWS) + .bind("days", STALE_STATS_DAYS) + .map( + (rs, ctx) -> + new Finding( + DiagnosticCategory.STALE_STATS, + Severity.WARN, + Map.of( + "table", rs.getString("table_name"), + "live_rows", String.valueOf(rs.getLong("n_live_tup")), + "last_analyzed", String.valueOf(rs.getString("last_analyzed"))))) + .list(); + } + + List seqScanHeavy(final Handle handle) { + return handle + .createQuery( + "SELECT relname AS table_name, seq_scan, idx_scan " + + "FROM pg_stat_user_tables " + + "WHERE seq_scan > :min_seq " + + " AND seq_scan::numeric / NULLIF(idx_scan, 0) > :ratio " + + "ORDER BY seq_scan DESC " + + "LIMIT 25") + .bind("min_seq", SEQ_SCAN_MIN) + .bind("ratio", SEQ_SCAN_RATIO) + .map( + (rs, ctx) -> + new Finding( + DiagnosticCategory.SEQ_SCAN_HEAVY, + Severity.INFO, + Map.of( + "table", rs.getString("table_name"), + "seq_scans", String.valueOf(rs.getLong("seq_scan")), + "idx_scans", String.valueOf(rs.getLong("idx_scan")), + "ratio", + rs.getLong("idx_scan") == 0 + ? "∞" + : String.valueOf(rs.getLong("seq_scan") / rs.getLong("idx_scan"))))) + .list(); + } + + List slowQueries(final Handle handle, final List notes) { + if (!hasPgStatStatements(handle)) { + notes.add("slow queries: pg_stat_statements extension not installed"); + return List.of(); + } + return handle + .createQuery( + "SELECT query, calls, mean_exec_time AS mean_ms " + + "FROM pg_stat_statements " + + "WHERE calls > :min_calls " + + "ORDER BY mean_exec_time DESC " + + "LIMIT :limit") + .bind("min_calls", SLOW_QUERY_MIN_CALLS) + .bind("limit", SLOW_QUERY_LIMIT) + .map( + (rs, ctx) -> { + Map attrs = new LinkedHashMap<>(); + attrs.put("query", truncate(rs.getString("query"))); + attrs.put("calls", String.valueOf(rs.getLong("calls"))); + attrs.put( + "mean_ms", String.format(java.util.Locale.ROOT, "%.1f", rs.getDouble("mean_ms"))); + return new Finding(DiagnosticCategory.SLOW_QUERY, Severity.INFO, attrs); + }) + .list(); + } + + private boolean hasPgStatStatements(final Handle handle) { + return handle + .createQuery("SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements'") + .mapTo(Integer.class) + .findOne() + .isPresent(); + } + + static String truncate(final String query) { + if (query == null) { + return ""; + } + String collapsed = query.replaceAll("\\s+", " ").trim(); + return collapsed.length() <= QUERY_TRUNCATE + ? collapsed + : collapsed.substring(0, QUERY_TRUNCATE) + "…"; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/PostgresTuningCatalog.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/PostgresTuningCatalog.java new file mode 100644 index 000000000000..df2b7e0e7705 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/PostgresTuningCatalog.java @@ -0,0 +1,143 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +/** + * Static catalog of which tables get which Postgres autovacuum reloptions, and the row-count + * threshold below which we skip tuning that table (small dev installs don't need aggressive + * autovacuum). Values come from production analysis of the 600k-container tenant. + */ +final class PostgresTuningCatalog { + + static final String AUTOVACUUM_VACUUM_SCALE_FACTOR = "autovacuum_vacuum_scale_factor"; + static final String AUTOVACUUM_ANALYZE_SCALE_FACTOR = "autovacuum_analyze_scale_factor"; + static final String AUTOVACUUM_VACUUM_COST_LIMIT = "autovacuum_vacuum_cost_limit"; + static final String AUTOVACUUM_VACUUM_COST_DELAY = "autovacuum_vacuum_cost_delay"; + + /** A tuning recipe for one table. */ + record Profile(Map settings, long rowThreshold, boolean relax, String reason) { + Profile { + settings = Map.copyOf(settings); + } + } + + private static final Map HOT_RELATIONSHIP = + Map.of( + AUTOVACUUM_ANALYZE_SCALE_FACTOR, "0.005", + AUTOVACUUM_VACUUM_SCALE_FACTOR, "0.01", + AUTOVACUUM_VACUUM_COST_LIMIT, "4000"); + + private static final Map HOT_TAG_USAGE = + Map.of( + AUTOVACUUM_ANALYZE_SCALE_FACTOR, "0.005", + AUTOVACUUM_VACUUM_SCALE_FACTOR, "0.01", + AUTOVACUUM_VACUUM_COST_LIMIT, "4000", + AUTOVACUUM_VACUUM_COST_DELAY, "0"); + + private static final Map ENTITY_LARGE = + Map.of( + AUTOVACUUM_ANALYZE_SCALE_FACTOR, "0.01", + AUTOVACUUM_VACUUM_SCALE_FACTOR, "0.02"); + + private static final Map ENTITY_SERVICE = + Map.of( + AUTOVACUUM_ANALYZE_SCALE_FACTOR, "0.02", + AUTOVACUUM_VACUUM_SCALE_FACTOR, "0.05"); + + private static final Map APPEND_ONLY = + Map.of( + AUTOVACUUM_ANALYZE_SCALE_FACTOR, "0.1", + AUTOVACUUM_VACUUM_SCALE_FACTOR, "0.2"); + + private static final long ROW_THRESHOLD_HOT = 0; + private static final long ROW_THRESHOLD_ENTITY_LARGE = 10_000; + private static final long ROW_THRESHOLD_ENTITY_SERVICE = 5_000; + private static final long ROW_THRESHOLD_APPEND_ONLY = 50_000; + + private static final Map CATALOG = buildCatalog(); + + private PostgresTuningCatalog() {} + + static Map catalog() { + return CATALOG; + } + + static Set tableNames() { + return CATALOG.keySet(); + } + + static Profile profileFor(final String tableName) { + return CATALOG.get(tableName); + } + + private static Map buildCatalog() { + Map map = new LinkedHashMap<>(); + map.put( + "entity_relationship", + new Profile(HOT_RELATIONSHIP, ROW_THRESHOLD_HOT, false, "Join target, write-heavy")); + map.put( + "tag_usage", + new Profile(HOT_TAG_USAGE, ROW_THRESHOLD_HOT, false, "Hottest table on read path")); + addEntityLarge(map); + addEntityService(map); + map.put( + "change_event", + new Profile(APPEND_ONLY, ROW_THRESHOLD_APPEND_ONLY, true, "Append-only, relax autovacuum")); + return Map.copyOf(map); + } + + private static void addEntityLarge(final Map map) { + String reason = "Large entity table; tighten autovacuum so list count stats stay fresh"; + for (String t : + new String[] { + "storage_container_entity", + "table_entity", + "dashboard_entity", + "pipeline_entity", + "chart_entity", + "topic_entity", + "ml_model_entity", + "glossary_term_entity", + "metric_entity", + "report_entity", + "search_index_entity", + "api_collection_entity", + "api_endpoint_entity", + "dashboard_data_model_entity", + "ingestion_pipeline_entity", + "data_contract_entity", + "stored_procedure_entity", + "directory_entity", + "file_entity", + "spreadsheet_entity", + "worksheet_entity", + "query_entity" + }) { + map.put(t, new Profile(ENTITY_LARGE, ROW_THRESHOLD_ENTITY_LARGE, false, reason)); + } + } + + private static void addEntityService(final Map map) { + String reason = "Service-tier table; mild tightening"; + map.put( + "database_entity", + new Profile(ENTITY_SERVICE, ROW_THRESHOLD_ENTITY_SERVICE, false, reason)); + map.put( + "database_schema_entity", + new Profile(ENTITY_SERVICE, ROW_THRESHOLD_ENTITY_SERVICE, false, reason)); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/ServerParamCheck.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/ServerParamCheck.java new file mode 100644 index 000000000000..39142703e3bd --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/ServerParamCheck.java @@ -0,0 +1,30 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +public record ServerParamCheck( + String parameter, String currentValue, String recommendedValue, String status, String note) { + + public static final String STATUS_OK = "OK"; + + /** + * Direction-agnostic. Some recommended values (e.g. {@code random_page_cost = 1.1}, + * {@code autovacuum_*_scale_factor}) are deliberately lower than the engine default — labelling + * those mismatches as "undersized" would be wrong. Operators see the actual current vs + * recommended values in the report and can judge direction themselves. + */ + public static final String STATUS_MISMATCH = "MISMATCH"; + + public static final String STATUS_UNTUNED = "UNTUNED"; + public static final String STATUS_UNKNOWN = "UNKNOWN"; +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Severity.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Severity.java new file mode 100644 index 000000000000..05993ef669d4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/Severity.java @@ -0,0 +1,18 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +public enum Severity { + INFO, + WARN +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/TableRecommendation.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/TableRecommendation.java new file mode 100644 index 000000000000..ecf509dac9be --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/TableRecommendation.java @@ -0,0 +1,30 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.util.Map; + +public record TableRecommendation( + String tableName, + Action action, + long rowCount, + long totalBytes, + Map currentSettings, + Map recommendedSettings, + String reason) { + + public TableRecommendation { + currentSettings = currentSettings == null ? Map.of() : Map.copyOf(currentSettings); + recommendedSettings = recommendedSettings == null ? Map.of() : Map.copyOf(recommendedSettings); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/TableStats.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/TableStats.java new file mode 100644 index 000000000000..9a8d8ec4e421 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/dbtune/TableStats.java @@ -0,0 +1,31 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import java.util.Map; + +public record TableStats( + String tableName, + long rowCount, + long dataBytes, + long indexBytes, + Map currentSettings) { + + public TableStats { + currentSettings = currentSettings == null ? Map.of() : Map.copyOf(currentSettings); + } + + public long totalBytes() { + return dataBytes + indexBytes; + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/dbtune/DbTuneReportTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/dbtune/DbTuneReportTest.java new file mode 100644 index 000000000000..f3e4a254dbd1 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/dbtune/DbTuneReportTest.java @@ -0,0 +1,173 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class DbTuneReportTest { + + @Test + void formatBytes_handlesAllScales() { + assertEquals("0 B", DbTuneReport.formatBytes(0)); + assertEquals("512 B", DbTuneReport.formatBytes(512)); + assertEquals("2 KB", DbTuneReport.formatBytes(2048)); + assertEquals("4 MB", DbTuneReport.formatBytes(4L * 1024 * 1024)); + assertEquals("1.5 GB", DbTuneReport.formatBytes((long) (1.5 * 1024 * 1024 * 1024))); + } + + @Test + void formatSettings_emptyOrNullShowsDefault() { + assertEquals("(default)", DbTuneReport.formatSettings(null)); + assertEquals("(default)", DbTuneReport.formatSettings(Map.of())); + } + + @Test + void formatSettings_sortsKeysAlphabetically() { + String formatted = + DbTuneReport.formatSettings( + Map.of( + "autovacuum_vacuum_scale_factor", "0.02", + "autovacuum_analyze_scale_factor", "0.01")); + + assertEquals( + "autovacuum_analyze_scale_factor=0.01, autovacuum_vacuum_scale_factor=0.02", formatted); + } + + @Test + void render_includesEngineAndAllSections() { + DbTuneResult result = + new DbTuneResult( + "PostgreSQL", + "17.2", + List.of( + new ServerParamCheck( + "shared_buffers", "16384", "40% of RAM", ServerParamCheck.STATUS_UNTUNED, "")), + List.of( + new TableRecommendation( + "storage_container_entity", + Action.APPLY, + 580_000, + 2L * 1024 * 1024 * 1024, + Map.of(), + Map.of("autovacuum_vacuum_scale_factor", "0.02"), + "Large entity table"))); + + String report = DbTuneReport.render(result); + + assertTrue(report.contains("PostgreSQL 17.2")); + assertTrue(report.contains("Server-level parameter compliance")); + assertTrue(report.contains("Per-table recommendations")); + assertTrue(report.contains("storage_container_entity")); + assertTrue(report.contains("APPLY")); + assertTrue(report.contains("Next steps:")); + } + + @Test + void render_zeroRecommendationsSuppressesAllMatchAndNextSteps() { + DbTuneResult result = new DbTuneResult("PostgreSQL", "17.2", List.of(), List.of()); + + String report = DbTuneReport.render(result); + + assertTrue(report.contains("none of the tracked tables exist")); + assertFalse( + report.contains("already match their recommended settings"), + "Empty recommendations must not claim everything matches"); + assertFalse(report.contains("Next steps:")); + } + + @Test + void render_noActionableShowsAllGoodMessage() { + DbTuneResult result = + new DbTuneResult( + "PostgreSQL", + "17.2", + List.of(), + List.of( + new TableRecommendation( + "storage_container_entity", + Action.OK, + 580_000, + 1_000_000L, + Map.of("autovacuum_vacuum_scale_factor", "0.02"), + Map.of("autovacuum_vacuum_scale_factor", "0.02"), + "ok"))); + + String report = DbTuneReport.render(result); + + assertTrue(report.contains("already match their recommended settings")); + assertFalse(report.contains("Next steps:")); + } + + @Test + void renderAlterStatements_emitsOneSemicolonPerStatement() { + PostgresAutoTuner tuner = new PostgresAutoTuner(); + TableRecommendation a = + new TableRecommendation( + "table_entity", + Action.APPLY, + 500_000, + 1_000_000L, + Map.of(), + Map.of("autovacuum_vacuum_scale_factor", "0.02"), + "ok"); + TableRecommendation b = + new TableRecommendation( + "dashboard_entity", + Action.APPLY, + 300_000, + 1_000_000L, + Map.of(), + Map.of("autovacuum_vacuum_scale_factor", "0.02"), + "ok"); + + String out = DbTuneReport.renderAlterStatements(tuner, List.of(a, b)); + + String[] lines = out.split("\n"); + assertEquals(2, lines.length); + assertTrue(lines[0].endsWith(";")); + assertTrue(lines[1].endsWith(";")); + assertTrue(lines[0].contains("table_entity")); + assertTrue(lines[1].contains("dashboard_entity")); + } + + @Test + void actionableRecommendations_excludesOkAndSkip() { + DbTuneResult result = + new DbTuneResult( + "PostgreSQL", + "17", + List.of(), + List.of( + rec("a", Action.APPLY), + rec("b", Action.OK), + rec("c", Action.SKIP), + rec("d", Action.TIGHTEN), + rec("e", Action.RELAX))); + + List actionable = result.actionableRecommendations(); + + assertEquals(3, actionable.size()); + assertEquals( + List.of("a", "d", "e"), actionable.stream().map(TableRecommendation::tableName).toList()); + } + + private static TableRecommendation rec(final String name, final Action action) { + return new TableRecommendation(name, action, 0, 0, Map.of(), Map.of(), ""); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/dbtune/DiagnosticReportTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/dbtune/DiagnosticReportTest.java new file mode 100644 index 000000000000..daf6a6669d3f --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/dbtune/DiagnosticReportTest.java @@ -0,0 +1,153 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Diagnostic-side rendering and grouping tests. Pure logic, no DB. The end-to-end DB query + * exercise lives in {@code DbTuneIT}. + */ +class DiagnosticReportTest { + + @Test + void findingsByCategory_groupsByEnumOrder() { + DbTuneDiagnosis d = + new DbTuneDiagnosis( + List.of( + finding(DiagnosticCategory.SLOW_QUERY, "q1"), + finding(DiagnosticCategory.UNUSED_INDEX, "idx_a"), + finding(DiagnosticCategory.UNUSED_INDEX, "idx_b"), + finding(DiagnosticCategory.HIGH_DEAD_TUPLES, "tag_usage")), + List.of()); + + Map> grouped = d.findingsByCategory(); + + assertEquals(2, grouped.get(DiagnosticCategory.UNUSED_INDEX).size()); + assertEquals(1, grouped.get(DiagnosticCategory.HIGH_DEAD_TUPLES).size()); + assertEquals(1, grouped.get(DiagnosticCategory.SLOW_QUERY).size()); + // EnumMap preserves enum declaration order — UNUSED_INDEX precedes HIGH_DEAD_TUPLES precedes + // SLOW_QUERY. + List orderedKeys = grouped.keySet().stream().toList(); + assertEquals( + List.of( + DiagnosticCategory.UNUSED_INDEX, + DiagnosticCategory.HIGH_DEAD_TUPLES, + DiagnosticCategory.SLOW_QUERY), + orderedKeys); + } + + @Test + void renderDiagnosis_empty_showsCleanResultMessage() { + DbTuneDiagnosis empty = new DbTuneDiagnosis(List.of(), List.of()); + + String out = DbTuneReport.renderDiagnosis(empty); + + assertTrue(out.contains("Diagnostic findings")); + assertTrue(out.contains("every check returned a clean result")); + } + + @Test + void renderDiagnosis_findingsRenderUnderCategorySections() { + DbTuneDiagnosis d = + new DbTuneDiagnosis( + List.of( + new Finding( + DiagnosticCategory.UNUSED_INDEX, + Severity.WARN, + Map.of( + "table", "tag_usage", + "index", "idx_unused_tag", + "size", "120 MB", + "scans", "0"))), + List.of()); + + String out = DbTuneReport.renderDiagnosis(d); + + assertTrue(out.contains("Unused indexes (1 found)")); + assertTrue(out.contains("idx_unused_tag")); + assertTrue(out.contains("120 MB")); + } + + @Test + void renderDiagnosis_notesAppendedWhenPresent() { + DbTuneDiagnosis d = + new DbTuneDiagnosis( + List.of(), List.of("slow queries: pg_stat_statements extension not installed")); + + String out = DbTuneReport.renderDiagnosis(d); + + assertTrue(out.contains("Notes:")); + assertTrue(out.contains("pg_stat_statements extension not installed")); + } + + @Test + void renderDiagnosis_categoriesWithoutFindingsAreSuppressed() { + DbTuneDiagnosis d = + new DbTuneDiagnosis(List.of(finding(DiagnosticCategory.SLOW_QUERY, "SELECT 1")), List.of()); + + String out = DbTuneReport.renderDiagnosis(d); + + assertTrue(out.contains("Top slowest queries")); + assertFalse(out.contains("Unused indexes")); + assertFalse(out.contains("Tables with high dead-tuple ratio")); + } + + @Test + void truncate_collapsesWhitespaceAndAppliesLimit() { + String long_ = + "SELECT *\nFROM table_entity\nWHERE fqnHash LIKE 'foo%' ORDER BY name LIMIT 100"; + + String t = PostgresDiagnostic.truncate(long_); + + assertFalse(t.contains(" ")); + assertFalse(t.contains("\n")); + assertTrue(t.length() <= 101); // 100 + ellipsis + } + + @Test + void truncate_nullReturnsEmpty() { + assertEquals("", PostgresDiagnostic.truncate(null)); + assertEquals("", MysqlDiagnostic.truncate(null)); + } + + @Test + void truncate_underLimitReturnsAsIs() { + assertEquals("SELECT 1", PostgresDiagnostic.truncate("SELECT 1")); + } + + @Test + void truncate_overLimitGetsEllipsis() { + String long_ = "x".repeat(150); + String t = PostgresDiagnostic.truncate(long_); + assertTrue(t.endsWith("…")); + assertEquals(101, t.length()); + } + + @Test + void diagnosticCategory_columnsAreImmutable() { + List cols = DiagnosticCategory.UNUSED_INDEX.columns(); + org.junit.jupiter.api.Assertions.assertThrows( + UnsupportedOperationException.class, () -> cols.add("new_col")); + } + + private static Finding finding(final DiagnosticCategory category, final String objectName) { + return new Finding(category, Severity.INFO, Map.of("table", objectName)); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/dbtune/MysqlAutoTunerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/dbtune/MysqlAutoTunerTest.java new file mode 100644 index 000000000000..22e73875b70a --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/dbtune/MysqlAutoTunerTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Heuristic-only tests for the MySQL tuner. Mirrors {@link PostgresAutoTunerTest} but pins the + * MySQL-specific reloption keys (STATS_PERSISTENT / STATS_AUTO_RECALC / STATS_SAMPLE_PAGES) and + * ALTER TABLE syntax (no parens, comma-separated key=value). + */ +class MysqlAutoTunerTest { + + private final MysqlAutoTuner tuner = new MysqlAutoTuner(); + + @Test + void recommend_unknownTable_returnsSkip() { + TableStats stats = stats("not_a_real_table", 1_000_000, Map.of()); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.SKIP, rec.action()); + } + + @Test + void recommend_belowRowThreshold_returnsSkip() { + TableStats stats = stats("storage_container_entity", 100, Map.of()); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.SKIP, rec.action()); + } + + @Test + void recommend_largeEntityWithNoSettings_returnsApply() { + TableStats stats = stats("storage_container_entity", 580_000, Map.of()); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.APPLY, rec.action()); + assertEquals("64", rec.recommendedSettings().get("STATS_SAMPLE_PAGES")); + assertEquals("1", rec.recommendedSettings().get("STATS_PERSISTENT")); + } + + @Test + void recommend_hotTablesGetHigherSampling() { + TableStats stats = stats("tag_usage", 7_400_000, Map.of()); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.APPLY, rec.action()); + assertEquals("100", rec.recommendedSettings().get("STATS_SAMPLE_PAGES")); + } + + @Test + void recommend_alreadyMatching_returnsOk() { + TableStats stats = + stats( + "storage_container_entity", + 580_000, + Map.of( + "STATS_PERSISTENT", "1", + "STATS_AUTO_RECALC", "1", + "STATS_SAMPLE_PAGES", "64")); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.OK, rec.action()); + } + + @Test + void recommend_partialSettings_returnsTighten() { + TableStats stats = + stats("storage_container_entity", 580_000, Map.of("STATS_SAMPLE_PAGES", "20")); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.TIGHTEN, rec.action()); + } + + @Test + void buildAlterStatement_usesMySqlSyntax() { + TableRecommendation rec = + new TableRecommendation( + "storage_container_entity", + Action.APPLY, + 500_000, + 1_000_000_000L, + Map.of(), + Map.of( + "STATS_PERSISTENT", "1", + "STATS_AUTO_RECALC", "1", + "STATS_SAMPLE_PAGES", "64"), + "ok"); + + String sql = tuner.buildAlterStatement(rec); + + assertEquals( + "ALTER TABLE `storage_container_entity` " + + "STATS_AUTO_RECALC=1, STATS_PERSISTENT=1, STATS_SAMPLE_PAGES=64", + sql); + } + + @Test + void parseCreateOptions_emptyAndBlankProduceEmptyMap() { + assertTrue(MysqlAutoTuner.parseCreateOptions(null).isEmpty()); + assertTrue(MysqlAutoTuner.parseCreateOptions("").isEmpty()); + assertTrue(MysqlAutoTuner.parseCreateOptions(" ").isEmpty()); + } + + @Test + void parseCreateOptions_extractsOnlyStatsKeys() { + Map parsed = + MysqlAutoTuner.parseCreateOptions( + "row_format=DYNAMIC stats_persistent=1 stats_sample_pages=64"); + + assertEquals(Map.of("STATS_PERSISTENT", "1", "STATS_SAMPLE_PAGES", "64"), parsed); + } + + @Test + void quoteIdent_usesBacktickAndRejectsUnsafe() { + assertEquals( + "`storage_container_entity`", MysqlAutoTuner.quoteIdent("storage_container_entity")); + assertThrows(IllegalArgumentException.class, () -> MysqlAutoTuner.quoteIdent("`evil`")); + assertThrows(IllegalArgumentException.class, () -> MysqlAutoTuner.quoteIdent("foo;bar")); + } + + @Test + void buildServerCheck_recommendedFormulaIsUntuned() { + ServerParamCheck check = + MysqlAutoTuner.buildServerCheck("innodb_buffer_pool_size", "1073741824", "40-60% of RAM"); + + assertEquals(ServerParamCheck.STATUS_UNTUNED, check.status()); + } + + private static TableStats stats( + final String tableName, final long rowCount, final Map currentSettings) { + return new TableStats(tableName, rowCount, 1_000_000L, 500_000L, currentSettings); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/dbtune/PostgresAutoTunerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/dbtune/PostgresAutoTunerTest.java new file mode 100644 index 000000000000..3dac30c63ce9 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/dbtune/PostgresAutoTunerTest.java @@ -0,0 +1,251 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util.dbtune; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Heuristic-only tests — no database. Walks the {@code recommend(stats) -> recommendation} pure + * function across the six action outcomes: SKIP for unknown table, SKIP under threshold, APPLY for + * empty-and-tighten, OK for already-matching, TIGHTEN for partial-match, RELAX for change_event. + * Also pins the SQL-builder format and identifier-quoting safety invariants because both feed + * directly into ALTER TABLE statements. + */ +class PostgresAutoTunerTest { + + private final PostgresAutoTuner tuner = new PostgresAutoTuner(); + + @Test + void recommend_unknownTable_returnsSkip() { + TableStats stats = stats("not_a_real_table", 1_000_000, Map.of()); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.SKIP, rec.action()); + assertTrue(rec.reason().contains("not in the dbtune catalog")); + } + + @Test + void recommend_belowRowThreshold_returnsSkip() { + TableStats stats = stats("storage_container_entity", 50, Map.of()); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.SKIP, rec.action()); + assertTrue(rec.reason().contains("below threshold")); + } + + @Test + void recommend_largeEntityWithNoSettings_returnsApply() { + TableStats stats = stats("storage_container_entity", 580_000, Map.of()); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.APPLY, rec.action()); + assertEquals("0.01", rec.recommendedSettings().get("autovacuum_analyze_scale_factor")); + assertEquals("0.02", rec.recommendedSettings().get("autovacuum_vacuum_scale_factor")); + } + + @Test + void recommend_largeEntityWithLooserSettings_returnsTighten() { + TableStats stats = + stats( + "storage_container_entity", + 580_000, + Map.of( + "autovacuum_analyze_scale_factor", "0.1", + "autovacuum_vacuum_scale_factor", "0.2")); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.TIGHTEN, rec.action()); + } + + @Test + void recommend_alreadyMatching_returnsOk() { + TableStats stats = + stats( + "storage_container_entity", + 580_000, + Map.of( + "autovacuum_analyze_scale_factor", "0.01", + "autovacuum_vacuum_scale_factor", "0.02")); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.OK, rec.action()); + } + + @Test + void recommend_alreadyMatchingNumericallyDifferentTextually_returnsOk() { + TableStats stats = + stats( + "storage_container_entity", + 580_000, + Map.of( + "autovacuum_analyze_scale_factor", "0.010", + "autovacuum_vacuum_scale_factor", "0.0200")); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.OK, rec.action(), "0.010 must equal 0.01 numerically"); + } + + @Test + void recommend_changeEventWithNoSettings_returnsRelax() { + TableStats stats = stats("change_event", 12_000_000, Map.of()); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.RELAX, rec.action()); + assertEquals("0.1", rec.recommendedSettings().get("autovacuum_analyze_scale_factor")); + assertEquals("0.2", rec.recommendedSettings().get("autovacuum_vacuum_scale_factor")); + } + + @Test + void recommend_hotTableHasZeroThreshold() { + TableStats stats = stats("entity_relationship", 1, Map.of()); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.APPLY, rec.action()); + assertEquals("4000", rec.recommendedSettings().get("autovacuum_vacuum_cost_limit")); + } + + @Test + void recommend_tagUsageRecommendsCostDelayZero() { + TableStats stats = stats("tag_usage", 7_400_000, Map.of()); + + TableRecommendation rec = tuner.recommend(stats); + + assertEquals(Action.APPLY, rec.action()); + assertEquals("0", rec.recommendedSettings().get("autovacuum_vacuum_cost_delay")); + } + + @Test + void buildAlterStatement_emitsSortedKeyValuePairs() { + TableRecommendation rec = + new TableRecommendation( + "storage_container_entity", + Action.APPLY, + 500_000, + 1_000_000_000L, + Map.of(), + Map.of( + "autovacuum_analyze_scale_factor", "0.01", + "autovacuum_vacuum_scale_factor", "0.02"), + "ok"); + + String sql = tuner.buildAlterStatement(rec); + + assertEquals( + "ALTER TABLE \"storage_container_entity\" SET (" + + "autovacuum_analyze_scale_factor = 0.01, " + + "autovacuum_vacuum_scale_factor = 0.02)", + sql); + } + + @Test + void parseReloptions_emptyAndNullProduceEmptyMap() { + assertTrue(PostgresAutoTuner.parseReloptions(null).isEmpty()); + assertTrue(PostgresAutoTuner.parseReloptions(new String[0]).isEmpty()); + } + + @Test + void parseReloptions_filtersUnknownKeysAndLowercasesNames() { + Map parsed = + PostgresAutoTuner.parseReloptions( + new String[] {"AUTOVACUUM_VACUUM_SCALE_FACTOR=0.05", "fillfactor=90"}); + + assertEquals(Map.of("autovacuum_vacuum_scale_factor", "0.05"), parsed); + } + + @Test + void quoteIdent_rejectsSqlInjectionAttempts() { + assertThrows( + IllegalArgumentException.class, () -> PostgresAutoTuner.quoteIdent("foo; DROP TABLE bar")); + assertThrows(IllegalArgumentException.class, () -> PostgresAutoTuner.quoteIdent("\"oops\"")); + } + + @Test + void quoteIdent_acceptsValidIdentifiers() { + assertEquals( + "\"storage_container_entity\"", PostgresAutoTuner.quoteIdent("storage_container_entity")); + } + + @Test + void settingsMatch_recommendedSubsetOfCurrent_isMatch() { + Map rec = Map.of("a", "0.01"); + Map current = Map.of("a", "0.01", "b", "999"); + + assertTrue(PostgresAutoTuner.settingsMatch(current, rec)); + } + + @Test + void settingsMatch_missingRecommendedKey_isNotMatch() { + Map rec = Map.of("a", "0.01", "b", "0.02"); + Map current = Map.of("a", "0.01"); + + assertFalse(PostgresAutoTuner.settingsMatch(current, rec)); + } + + @Test + void buildServerCheck_recommendedFormulaIsUntuned() { + ServerParamCheck check = + PostgresAutoTuner.buildServerCheck("shared_buffers", "16384", "40% of RAM"); + + assertEquals(ServerParamCheck.STATUS_UNTUNED, check.status()); + } + + @Test + void buildServerCheck_currentMissingIsUnknown() { + ServerParamCheck check = PostgresAutoTuner.buildServerCheck("missing", null, "200"); + + assertEquals(ServerParamCheck.STATUS_UNKNOWN, check.status()); + } + + @Test + void buildServerCheck_numericMatchIsOk() { + ServerParamCheck check = PostgresAutoTuner.buildServerCheck("random_page_cost", "1.10", "1.1"); + + assertEquals(ServerParamCheck.STATUS_OK, check.status()); + } + + @Test + void buildServerCheck_numericMismatchIsLabelledMismatch() { + ServerParamCheck check = PostgresAutoTuner.buildServerCheck("work_mem", "4096", "131072"); + + assertEquals(ServerParamCheck.STATUS_MISMATCH, check.status()); + } + + @Test + void buildServerCheck_currentHigherThanRecommendedIsAlsoMismatch() { + // random_page_cost recommendation (1.1) is intentionally LOWER than the SSD-naive default + // (4.0). + // Direction-agnostic MISMATCH avoids the misleading "UNDERSIZED" label here. + ServerParamCheck check = PostgresAutoTuner.buildServerCheck("random_page_cost", "4.0", "1.1"); + + assertEquals(ServerParamCheck.STATUS_MISMATCH, check.status()); + } + + private static TableStats stats( + final String tableName, final long rowCount, final Map currentSettings) { + return new TableStats(tableName, rowCount, 1_000_000L, 500_000L, currentSettings); + } +}