diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/CompactionCleaner.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/CompactionCleaner.java index fd49bbb90f13..2b1e7a7f19c3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/CompactionCleaner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/CompactionCleaner.java @@ -158,6 +158,7 @@ private void clean(CompactionInfo ci, long minOpenTxn, boolean metricsEnabled) t cleanUsingAcidDir(ci, t, path, cleanerWaterMark); } } else { + ci.setSoftDelete(true); cleanUsingLocation(ci, location, false); } } catch (Exception e) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfo.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfo.java index e1c1f492bab8..5c392477c6a0 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfo.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfo.java @@ -87,6 +87,7 @@ public class CompactionInfo implements Comparable { private String fullPartitionName = null; private String fullTableName = null; private StringableMap propertiesMap; + private boolean softDelete; public CompactionInfo(String dbname, String tableName, String partName, CompactionType type) { this.dbname = dbname; @@ -190,6 +191,7 @@ public String toString() { .append("numberOfBuckets", numberOfBuckets) .append("orderByClause", orderByClause) .append("minOpenWriteTxnId", minOpenWriteTxnId) + .append("softDelete", softDelete) .build(); } @@ -368,4 +370,12 @@ public void setWriteIds(boolean hasUncompactedAborts, Set writeIds) { public boolean isAbortedTxnCleanup() { return type == CompactionType.ABORT_TXN_CLEANUP; } + + public void setSoftDelete(boolean softDelete) { + this.softDelete = softDelete; + } + + public boolean isSoftDelete() { + return softDelete; + } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/MarkCleanedFunction.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/MarkCleanedFunction.java index 2bd96f53a1da..e1d9badf5683 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/MarkCleanedFunction.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/MarkCleanedFunction.java @@ -50,7 +50,7 @@ public MarkCleanedFunction(CompactionInfo info) { public Void execute(MultiDataSourceJdbcResource jdbcResource) throws MetaException { NamedParameterJdbcTemplate jdbcTemplate = jdbcResource.getJdbcTemplate(); MapSqlParameterSource param; - if (!info.isAbortedTxnCleanup()) { + if (!info.isAbortedTxnCleanup() && !info.isSoftDelete()) { param = new MapSqlParameterSource() .addValue("id", info.id) .addValue("succeeded", Character.toString(SUCCEEDED_STATE), Types.CHAR); @@ -85,6 +85,10 @@ public Void execute(MultiDataSourceJdbcResource jdbcResource) throws MetaExcepti */ removeCompactionAndAbortRetryEntries(info, jdbcTemplate); + if (info.isSoftDelete()) { + return null; + } + if (!info.isAbortedTxnCleanup()) { // Remove entries from completed_txn_components as well, so we don't start looking there // again but only up to the highest write ID include in this compaction job. @@ -175,7 +179,7 @@ private void removeCompactionAndAbortRetryEntries(CompactionInfo info, NamedPara String deleteQuery = """ DELETE FROM "COMPACTION_QUEUE" WHERE "CQ_ID" = :id """; - if (!info.isAbortedTxnCleanup()) { + if (!info.isAbortedTxnCleanup() && !info.isSoftDelete()) { deleteQuery += """ OR ("CQ_DATABASE" = :db AND "CQ_TABLE" = :table AND (:partition is NULL OR "CQ_PARTITION" = :partition)