Skip to content

fix: Move version metadata migration to 2.0.0 and remove crashing MySQL multi-valued index #27816

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
yan-3005 wants to merge 4 commits into from
Closed

fix: Move version metadata migration to 2.0.0 and remove crashing MySQL multi-valued index #27816

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
338fdf8
fix: move version metadata migration from 1.12.7 to 2.0.0 and remove …
yan-3005 Apr 29, 2026
7c3afeb
fix: null guard for id/extension in backfillVersionMetadata, remove d…
yan-3005 Apr 29, 2026
322436a
perf: cursor-based pagination + PreparedBatch for backfillVersionMeta…
yan-3005 Apr 29, 2026
14cb78e
Merge branch 'main' into ram/fix-version-metadata-migration
yan-3005 Apr 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 0 additions & 57 deletions bootstrap/sql/migrations/native/1.12.7/mysql/postDataMigrationSQLScript.sql
View file
Open in desktop

This file was deleted.

9 changes: 0 additions & 9 deletions bootstrap/sql/migrations/native/1.12.7/mysql/schemaChanges.sql
View file
Open in desktop

This file was deleted.

49 changes: 0 additions & 49 deletions bootstrap/sql/migrations/native/1.12.7/postgres/postDataMigrationSQLScript.sql
View file
Open in desktop

This file was deleted.

11 changes: 0 additions & 11 deletions bootstrap/sql/migrations/native/1.12.7/postgres/schemaChanges.sql
View file
Open in desktop

This file was deleted.

7 changes: 7 additions & 0 deletions bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,3 +135,10 @@ CREATE TABLE IF NOT EXISTS task_form_schema_entity (
KEY idx_task_form_schema_task_type (taskType),
KEY idx_task_form_schema_deleted (deleted)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;

ALTER TABLE entity_extension
ADD COLUMN versionNum DOUBLE NULL,
ADD COLUMN changedFieldKeys JSON NULL;
Comment thread
gitar-bot[bot] marked this conversation as resolved.

CREATE INDEX idx_entity_extension_version_order
ON entity_extension (id, versionNum);
Comment on lines +139 to +144
12 changes: 12 additions & 0 deletions bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,3 +140,15 @@ CREATE TABLE IF NOT EXISTS task_form_schema_entity (
CREATE INDEX IF NOT EXISTS idx_task_form_schema_name ON task_form_schema_entity (name);
CREATE INDEX IF NOT EXISTS idx_task_form_schema_tasktype ON task_form_schema_entity (tasktype);
CREATE INDEX IF NOT EXISTS idx_task_form_schema_deleted ON task_form_schema_entity (deleted);

ALTER TABLE entity_extension
ADD COLUMN IF NOT EXISTS versionNum DOUBLE PRECISION,
ADD COLUMN IF NOT EXISTS changedFieldKeys JSONB;

CREATE INDEX IF NOT EXISTS idx_entity_extension_version_order
ON entity_extension (id, versionNum DESC)
WHERE versionNum IS NOT NULL;

CREATE INDEX IF NOT EXISTS idx_entity_extension_changed_field_keys
ON entity_extension USING GIN (changedFieldKeys)
WHERE changedFieldKeys IS NOT NULL;
11 changes: 11 additions & 0 deletions ...tadata-service/src/main/java/org/openmetadata/service/migration/mysql/v200/Migration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,17 @@
import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL;
import static org.openmetadata.service.migration.utils.v200.MigrationUtil.addTableColumnSearchSettings;
import static org.openmetadata.service.migration.utils.v200.MigrationUtil.backfillAnnouncementRelationships;
import static org.openmetadata.service.migration.utils.v200.MigrationUtil.backfillVersionMetadata;
import static org.openmetadata.service.migration.utils.v200.MigrationUtil.migrateLegacyActivityThreadsToActivityStream;
import static org.openmetadata.service.migration.utils.v200.MigrationUtil.migrateSuggestionsToTaskEntity;
import static org.openmetadata.service.migration.utils.v200.MigrationUtil.migrateThreadTasksToTaskEntity;

import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.service.migration.api.MigrationProcessImpl;
import org.openmetadata.service.migration.utils.MigrationFile;

@Slf4j
public class Migration extends MigrationProcessImpl {

public Migration(MigrationFile migrationFile) {
Expand All @@ -25,5 +28,13 @@ public void runDataMigration() {
migrateThreadTasksToTaskEntity(handle, MYSQL);
migrateLegacyActivityThreadsToActivityStream(handle, MYSQL);
backfillAnnouncementRelationships(handle);
try {
backfillVersionMetadata(handle);
} catch (Exception e) {
LOG.error(
"Failed to backfill versionNum and changedFieldKeys in v200 migration. "
+ "Version timeline filtering may not work correctly until the migration is re-run.",
e);
}
}
}
11 changes: 11 additions & 0 deletions ...ata-service/src/main/java/org/openmetadata/service/migration/postgres/v200/Migration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,17 @@
import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES;
import static org.openmetadata.service.migration.utils.v200.MigrationUtil.addTableColumnSearchSettings;
import static org.openmetadata.service.migration.utils.v200.MigrationUtil.backfillAnnouncementRelationships;
import static org.openmetadata.service.migration.utils.v200.MigrationUtil.backfillVersionMetadata;
import static org.openmetadata.service.migration.utils.v200.MigrationUtil.migrateLegacyActivityThreadsToActivityStream;
import static org.openmetadata.service.migration.utils.v200.MigrationUtil.migrateSuggestionsToTaskEntity;
import static org.openmetadata.service.migration.utils.v200.MigrationUtil.migrateThreadTasksToTaskEntity;

import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.service.migration.api.MigrationProcessImpl;
import org.openmetadata.service.migration.utils.MigrationFile;

@Slf4j
public class Migration extends MigrationProcessImpl {

public Migration(MigrationFile migrationFile) {
Expand All @@ -25,5 +28,13 @@ public void runDataMigration() {
migrateThreadTasksToTaskEntity(handle, POSTGRES);
migrateLegacyActivityThreadsToActivityStream(handle, POSTGRES);
backfillAnnouncementRelationships(handle);
try {
backfillVersionMetadata(handle);
} catch (Exception e) {
LOG.error(
"Failed to backfill versionNum and changedFieldKeys in v200 migration. "
+ "Version timeline filtering may not work correctly until the migration is re-run.",
e);
}
}
}
Loading
Loading