From 98c291367c74515c3a14486ebd72cc11ba70f0ea Mon Sep 17 00:00:00 2001 From: Alex Gaetano Padula Date: Tue, 7 Jul 2026 17:45:58 -0400 Subject: [PATCH 1/2] tidesdb engine correctness, statement atomicity, and planner-accuracy * update_row reuses the cached per-index dup-check iterator instead of building and tearing down an o(num_sstables) merge heap on every row, the same caching write_row already does.matters for a bulk update that shifts a unique-indexed column * restore statement-level atomicity inside begin...commit.a per-statement savepoint is armed in external_lock and, on statement error, we roll back to it and revert the plugin-side state to the same boundary -- txn_key_state through an undo journal, fts meta through a snapshot, and the statement's row locks to a marker -- then bump the txn generation so cached iterators rebuild.the old path did a full transaction rollback and shifted the snapshot, losing every earlier statement's work.its cited savepoint deep-copy cost was wrong, tidesdb savepoints are o(1).falls back to full rollback when a bulk mid-commit or a user rollback-to-savepoint voids it * skip compression on an encrypted table's data column family.rows are encrypted before they reach the library and ciphertext does not compress, so running lz4 over it on every flush and compaction spent cpu for nothing. index cfs hold unencrypted keys and keep their compression * advertise ha_keyread_only per key part.the covering bitmap is built by index_flags(idx, part), so a part is keyread-only only when its field can be rebuilt from its sort key.undecodable types (varchar, decimal, float, multi-byte char) no longer get priced as a row-fetch-free covering scan that then pays a hidden pk lookup per row.reading only the decodable columns of a mixed index still covers * seek to the upper bound for where seccol > x on a secondary index instead of stepping over every duplicate-prefix entry, with a single step past the one row whose pk encodes to all-0xff * probe pk uniqueness with tidesdb_txn_contains on the write txn and drop the per-row dedicated read-committed txn and its commit+reset churn.the non-tracking check keeps the probe out of the reservation base, and the insert's own reservation still enforces uniqueness at commit * take the row count from tidesdb_cf_estimate_cardinality, which dedupes the mvcc versions of a key within an sstable that total_keys counts separately and includes the shared unified memtable.fixes table_rows and index cardinality reading a floored minimum for unflushed tables * drop the misleading tdb_max_txn_ops line from the bulk-commit batching comments, the real bound is txn memory not the op cap * new mtr tests for statement atomicity, encrypted-cf compression, keyread decodability, secondary greater-than range, and row cardinality.re-record index_stats, info_schema, mrr, online_ddl and per_index_btree whose baselines had captured the old floored stats --- .../r/tidesdb_encrypted_no_compress.result | 64 +++ .../tidesdb/r/tidesdb_index_stats.result | 2 +- .../tidesdb/r/tidesdb_info_schema.result | 8 +- .../r/tidesdb_keyread_decodable.result | 111 ++++ mysql-test/suite/tidesdb/r/tidesdb_mrr.result | 2 +- .../suite/tidesdb/r/tidesdb_online_ddl.result | 18 +- .../tidesdb/r/tidesdb_per_index_btree.result | 6 +- .../tidesdb/r/tidesdb_row_cardinality.result | 51 ++ .../r/tidesdb_secondary_gt_range.result | 78 +++ .../tidesdb/r/tidesdb_stmt_atomicity.result | 162 ++++++ .../t/tidesdb_encrypted_no_compress.opt | 2 + .../t/tidesdb_encrypted_no_compress.test | 74 +++ .../tidesdb/t/tidesdb_keyread_decodable.test | 117 ++++ .../tidesdb/t/tidesdb_row_cardinality.test | 62 +++ .../tidesdb/t/tidesdb_secondary_gt_range.test | 65 +++ .../tidesdb/t/tidesdb_stmt_atomicity.test | 152 +++++ tidesdb/ha_tidesdb.cc | 525 +++++++++++++----- tidesdb/ha_tidesdb.h | 30 +- 18 files changed, 1368 insertions(+), 161 deletions(-) create mode 100644 mysql-test/suite/tidesdb/r/tidesdb_encrypted_no_compress.result create mode 100644 mysql-test/suite/tidesdb/r/tidesdb_keyread_decodable.result create mode 100644 mysql-test/suite/tidesdb/r/tidesdb_row_cardinality.result create mode 100644 mysql-test/suite/tidesdb/r/tidesdb_secondary_gt_range.result create mode 100644 mysql-test/suite/tidesdb/r/tidesdb_stmt_atomicity.result create mode 100644 mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.opt create mode 100644 mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.test create mode 100644 mysql-test/suite/tidesdb/t/tidesdb_keyread_decodable.test create mode 100644 mysql-test/suite/tidesdb/t/tidesdb_row_cardinality.test create mode 100644 mysql-test/suite/tidesdb/t/tidesdb_secondary_gt_range.test create mode 100644 mysql-test/suite/tidesdb/t/tidesdb_stmt_atomicity.test diff --git a/mysql-test/suite/tidesdb/r/tidesdb_encrypted_no_compress.result b/mysql-test/suite/tidesdb/r/tidesdb_encrypted_no_compress.result new file mode 100644 index 00000000..20410fc9 --- /dev/null +++ b/mysql-test/suite/tidesdb/r/tidesdb_encrypted_no_compress.result @@ -0,0 +1,64 @@ +# +# ============================================ +# Encrypted table: data CF compression forced off, index CF keeps it +# ============================================ +# +CREATE TABLE enc_lz4 ( +id INT PRIMARY KEY, +u INT, +v VARCHAR(200), +KEY k_u (u) +) ENGINE=TIDESDB `ENCRYPTED`=YES COMPRESSION='LZ4'; +# the table still reports the requested COMPRESSION option to the user +SHOW CREATE TABLE enc_lz4; +Table Create Table +enc_lz4 CREATE TABLE `enc_lz4` ( + `id` int(11) NOT NULL, + `u` int(11) DEFAULT NULL, + `v` varchar(200) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `k_u` (`u`) +) ENGINE=TidesDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci `ENCRYPTED`=YES `COMPRESSION`='LZ4' +# data CF: compression_algorithm = NONE (ciphertext is incompressible) +compression_algorithm = NONE +# index CF: compression_algorithm = LZ4 (unencrypted keys keep compression) +compression_algorithm = LZ4 +# +# ============================================ +# Control: non-encrypted table keeps the selected compression on its data CF +# ============================================ +# +CREATE TABLE plain_lz4 ( +id INT PRIMARY KEY, +v VARCHAR(200) +) ENGINE=TIDESDB COMPRESSION='LZ4'; +# data CF: compression_algorithm = LZ4 (not encrypted, so compression stays) +compression_algorithm = LZ4 +# +# ============================================ +# Data still round-trips under an encrypted, compression-disabled CF +# ============================================ +# +INSERT INTO enc_lz4 VALUES (1, 10, 'alpha'), (2, 20, 'beta'), (3, 30, 'gamma'); +UPDATE enc_lz4 SET v = 'delta' WHERE id = 2; +DELETE FROM enc_lz4 WHERE id = 3; +SELECT * FROM enc_lz4 ORDER BY id; +id u v +1 10 alpha +2 20 delta +# covering index scan on the still-compressed index CF +SELECT id FROM enc_lz4 WHERE u = 10; +id +1 +# +# ============================================ +# TRUNCATE recreates the data CF with compression still off +# ============================================ +# +TRUNCATE TABLE enc_lz4; +compression_algorithm = NONE +compression_algorithm = LZ4 +DROP TABLE enc_lz4; +DROP TABLE plain_lz4; +# +# Done. diff --git a/mysql-test/suite/tidesdb/r/tidesdb_index_stats.result b/mysql-test/suite/tidesdb/r/tidesdb_index_stats.result index 11aeb88a..8c4a0906 100644 --- a/mysql-test/suite/tidesdb/r/tidesdb_index_stats.result +++ b/mysql-test/suite/tidesdb/r/tidesdb_index_stats.result @@ -77,7 +77,7 @@ test.t_stats analyze status OK # After ANALYZE, the optimizer should estimate ~100 rows for k=0 EXPLAIN SELECT * FROM t_stats WHERE k = 0; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t_stats ref k_idx k_idx 4 const 2 +1 SIMPLE t_stats ALL k_idx NULL NULL NULL 200 Using where DROP TABLE t_stats; # # ============================================ diff --git a/mysql-test/suite/tidesdb/r/tidesdb_info_schema.result b/mysql-test/suite/tidesdb/r/tidesdb_info_schema.result index 716c0583..874a11fc 100644 --- a/mysql-test/suite/tidesdb/r/tidesdb_info_schema.result +++ b/mysql-test/suite/tidesdb/r/tidesdb_info_schema.result @@ -7,20 +7,20 @@ INSERT INTO t_info_schema VALUES (1, REPEAT('a', 100)); INSERT INTO t_info_schema VALUES (2, REPEAT('b', 100)); INSERT INTO t_info_schema VALUES (3, REPEAT('c', 100)); # =+=+= data_length must be non-zero =+=+= -FAIL: DATA_LENGTH is 0 +OK: DATA_LENGTH > 0 # =+=+= table_rows must reflect inserted rows =+=+= -FAIL: TABLE_ROWS < 3 +OK: TABLE_ROWS >= 3 # =+=+= add secondary index and check index_length =+=+= ALTER TABLE t_info_schema ADD INDEX idx_val (val); SELECT COUNT(*) FROM t_info_schema; COUNT(*) 3 -FAIL: INDEX_LENGTH is 0 +OK: INDEX_LENGTH > 0 # =+=+= verify after bulk insert =+=+= SELECT COUNT(*) FROM t_info_schema; COUNT(*) 200 -FAIL: DATA_LENGTH is 0 after bulk insert +OK: DATA_LENGTH > 0 after bulk insert # =+=+= create_time must be non-null =+=+= OK: CREATE_TIME is set # =+=+= update_time must be non-null after DML =+=+= diff --git a/mysql-test/suite/tidesdb/r/tidesdb_keyread_decodable.result b/mysql-test/suite/tidesdb/r/tidesdb_keyread_decodable.result new file mode 100644 index 00000000..8f0b09af --- /dev/null +++ b/mysql-test/suite/tidesdb/r/tidesdb_keyread_decodable.result @@ -0,0 +1,111 @@ +# +# ============================================ +# Decodable column types: covering (Using index) IS offered +# ============================================ +# +CREATE TABLE cov_ok ( +id INT PRIMARY KEY, +i INT, +b BIGINT, +d DATE, +ts DATETIME, +c CHAR(8) CHARACTER SET binary, +KEY k_i (i), +KEY k_b (b), +KEY k_d (d), +KEY k_ts (ts), +KEY k_c (c) +) ENGINE=TIDESDB; +INSERT INTO cov_ok VALUES +(1, 10, 100, '2020-01-01', '2020-01-01 10:00:00', 'aaa'), +(2, 20, 200, '2021-02-02', '2021-02-02 11:00:00', 'bbb'), +(3, 30, 300, '2022-03-03', '2022-03-03 12:00:00', 'ccc'); +EXPLAIN SELECT i FROM cov_ok FORCE INDEX (k_i) WHERE i = 20; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_ok ref k_i k_i 5 const # Using index +EXPLAIN SELECT b FROM cov_ok FORCE INDEX (k_b) WHERE b = 200; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_ok ref k_b k_b 9 const # Using index +EXPLAIN SELECT d FROM cov_ok FORCE INDEX (k_d) WHERE d = '2021-02-02'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_ok ref k_d k_d 4 const # Using index +EXPLAIN SELECT ts FROM cov_ok FORCE INDEX (k_ts) WHERE ts = '2021-02-02 11:00:00'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_ok ref k_ts k_ts 6 const # Using index +EXPLAIN SELECT c FROM cov_ok FORCE INDEX (k_c) WHERE c = 'bbb'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_ok ref k_c k_c 9 const # Using where; Using index +# +# ============================================ +# Undecodable column types: covering is NOT offered (no Using index) +# ============================================ +# +CREATE TABLE cov_no ( +id INT PRIMARY KEY, +s VARCHAR(50), +u CHAR(8) CHARACTER SET utf8mb4, +m DECIMAL(10,2), +f DOUBLE, +KEY k_s (s), +KEY k_u (u), +KEY k_m (m), +KEY k_f (f) +) ENGINE=TIDESDB; +INSERT INTO cov_no VALUES +(1, 'alpha', 'x1', 1.50, 1.5), +(2, 'beta', 'x2', 2.50, 2.5), +(3, 'gamma', 'x3', 3.50, 3.5); +EXPLAIN SELECT s FROM cov_no FORCE INDEX (k_s) WHERE s = 'beta'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_no ref k_s k_s 203 const # Using where +EXPLAIN SELECT u FROM cov_no FORCE INDEX (k_u) WHERE u = 'x2'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_no ref k_u k_u 33 const # Using where +EXPLAIN SELECT m FROM cov_no FORCE INDEX (k_m) WHERE m = 2.50; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_no ref k_m k_m 6 const # +EXPLAIN SELECT f FROM cov_no FORCE INDEX (k_f) WHERE f = 2.5; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_no ref k_f k_f 9 const # +# results are still correct on the non-covering (PK fetch) path +SELECT s FROM cov_no FORCE INDEX (k_s) WHERE s = 'beta'; +s +beta +SELECT m FROM cov_no FORCE INDEX (k_m) WHERE m = 2.50; +m +2.50 +# +# ============================================ +# Composite index: covering decided per key part, not per whole index +# ============================================ +# +CREATE TABLE cov_mix ( +id INT PRIMARY KEY, +a INT, +b INT, +s VARCHAR(50), +KEY k_ab (a, b), +KEY k_as (a, s) +) ENGINE=TIDESDB; +INSERT INTO cov_mix VALUES (1, 1, 10, 'x'), (2, 2, 20, 'y'), (3, 3, 30, 'z'); +# (a INT, b INT): both decodable -> Using index +EXPLAIN SELECT a, b FROM cov_mix FORCE INDEX (k_ab) WHERE a = 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_mix ref k_ab k_ab 5 const # Using index +# (a INT, s VARCHAR), reading only the decodable a -> still Using index +EXPLAIN SELECT a FROM cov_mix FORCE INDEX (k_as) WHERE a = 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_mix ref k_as k_as 5 const # Using index +# (a INT, s VARCHAR), reading the undecodable s -> no Using index +EXPLAIN SELECT a, s FROM cov_mix FORCE INDEX (k_as) WHERE a = 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_mix ref k_as k_as 5 const # +# result on the s (non-covering) path is still correct +SELECT a, s FROM cov_mix FORCE INDEX (k_as) WHERE a = 2; +a s +2 y +DROP TABLE cov_ok; +DROP TABLE cov_no; +DROP TABLE cov_mix; +# +# Done. diff --git a/mysql-test/suite/tidesdb/r/tidesdb_mrr.result b/mysql-test/suite/tidesdb/r/tidesdb_mrr.result index bea30b41..763cd938 100644 --- a/mysql-test/suite/tidesdb/r/tidesdb_mrr.result +++ b/mysql-test/suite/tidesdb/r/tidesdb_mrr.result @@ -9,7 +9,7 @@ INSERT INTO t_pk VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e'), # Confirm the optimizer actually picks Rowid-ordered scan (MRR). EXPLAIN SELECT * FROM t_pk WHERE id IN (7, 2, 9, 3, 5); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t_pk range PRIMARY # 4 NULL 2 Using where +1 SIMPLE t_pk range PRIMARY # 4 NULL 5 Using where # Unsorted IN-list; MRR must still return the right rows. SELECT * FROM t_pk WHERE id IN (7, 2, 9, 3, 5) ORDER BY id; id v diff --git a/mysql-test/suite/tidesdb/r/tidesdb_online_ddl.result b/mysql-test/suite/tidesdb/r/tidesdb_online_ddl.result index 2b8d42bb..b5c9b3d3 100644 --- a/mysql-test/suite/tidesdb/r/tidesdb_online_ddl.result +++ b/mysql-test/suite/tidesdb/r/tidesdb_online_ddl.result @@ -36,8 +36,8 @@ t_ddl CREATE TABLE `t_ddl` ( ALTER TABLE t_ddl ADD INDEX idx_a (a), ALGORITHM=INPLACE; SHOW INDEX FROM t_ddl; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored -t_ddl 0 PRIMARY 1 id A 2 NULL NULL LSM NO -t_ddl 1 idx_a 1 a A 2 NULL NULL YES LSM NO +t_ddl 0 PRIMARY 1 id A 6 NULL NULL LSM NO +t_ddl 1 idx_a 1 a A 6 NULL NULL YES LSM NO # Verify index is usable SELECT id, a FROM t_ddl WHERE a = 10 ORDER BY id; id a @@ -52,9 +52,9 @@ id a ALTER TABLE t_ddl ADD INDEX idx_c (c), ALGORITHM=INPLACE; SHOW INDEX FROM t_ddl; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored -t_ddl 0 PRIMARY 1 id A 2 NULL NULL LSM NO -t_ddl 1 idx_a 1 a A 2 NULL NULL YES LSM NO -t_ddl 1 idx_c 1 c A 2 NULL NULL YES LSM NO +t_ddl 0 PRIMARY 1 id A 6 NULL NULL LSM NO +t_ddl 1 idx_a 1 a A 6 NULL NULL YES LSM NO +t_ddl 1 idx_c 1 c A 6 NULL NULL YES LSM NO EXPLAIN SELECT id, c FROM t_ddl WHERE c = 200; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t_ddl ref idx_c idx_c 5 const 1 Using index @@ -65,8 +65,8 @@ id c ALTER TABLE t_ddl DROP INDEX idx_a, ALGORITHM=INPLACE; SHOW INDEX FROM t_ddl; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored -t_ddl 0 PRIMARY 1 id A 2 NULL NULL LSM NO -t_ddl 1 idx_c 1 c A 2 NULL NULL YES LSM NO +t_ddl 0 PRIMARY 1 id A 6 NULL NULL LSM NO +t_ddl 1 idx_c 1 c A 6 NULL NULL YES LSM NO # Verify remaining index still works SELECT id, c FROM t_ddl WHERE c = 300; id c @@ -75,8 +75,8 @@ id c ALTER TABLE t_ddl ADD INDEX idx_a2 (a), DROP INDEX idx_c, ALGORITHM=INPLACE; SHOW INDEX FROM t_ddl; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored -t_ddl 0 PRIMARY 1 id A 2 NULL NULL LSM NO -t_ddl 1 idx_a2 1 a A 2 NULL NULL YES LSM NO +t_ddl 0 PRIMARY 1 id A 6 NULL NULL LSM NO +t_ddl 1 idx_a2 1 a A 6 NULL NULL YES LSM NO EXPLAIN SELECT id, a FROM t_ddl WHERE a = 20; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t_ddl ref idx_a2 idx_a2 5 const 1 Using index diff --git a/mysql-test/suite/tidesdb/r/tidesdb_per_index_btree.result b/mysql-test/suite/tidesdb/r/tidesdb_per_index_btree.result index 3400c01d..0f80f9bd 100644 --- a/mysql-test/suite/tidesdb/r/tidesdb_per_index_btree.result +++ b/mysql-test/suite/tidesdb/r/tidesdb_per_index_btree.result @@ -12,9 +12,9 @@ INSERT INTO t1 VALUES (1,10,100),(2,20,200),(3,30,300); # idx_a should show BTREE, idx_b should show LSM SHOW KEYS FROM t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored -t1 0 PRIMARY 1 id A 2 NULL NULL LSM NO -t1 1 idx_a 1 a A 2 NULL NULL YES BTREE NO -t1 1 idx_b 1 b A 2 NULL NULL YES LSM NO +t1 0 PRIMARY 1 id A 3 NULL NULL LSM NO +t1 1 idx_a 1 a A 3 NULL NULL YES BTREE NO +t1 1 idx_b 1 b A 3 NULL NULL YES LSM NO SELECT * FROM t1 WHERE a = 20; id a b 2 20 200 diff --git a/mysql-test/suite/tidesdb/r/tidesdb_row_cardinality.result b/mysql-test/suite/tidesdb/r/tidesdb_row_cardinality.result new file mode 100644 index 00000000..017f072b --- /dev/null +++ b/mysql-test/suite/tidesdb/r/tidesdb_row_cardinality.result @@ -0,0 +1,51 @@ +CREATE TABLE t_a (id INT PRIMARY KEY, v INT, KEY k_v (v)) ENGINE=TIDESDB; +CREATE TABLE t_b (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB; +ANALYZE TABLE t_a; +Table Op Msg_type Msg_text +test.t_a analyze status Engine-independent statistics collected +test.t_a analyze Note [TIDESDB] CF 'test__t_a' total_keys=200 data_size=0 bytes memtable=4600 bytes levels=1 read_amp=1.00 cache_hit=0.0% +test.t_a analyze Note [TIDESDB] avg_key=6.9 bytes avg_value=16.1 bytes +test.t_a analyze Note [TIDESDB] level 1 sstables=0 size=0 bytes keys=0 +test.t_a analyze Note [TIDESDB] WA user=3800 wal=0 flush=0 (0 ssts) compact_write=0 (0 ssts) compact_read=0 ratio=0.00x +test.t_a analyze Note [TIDESDB] idx CF 'test__t_a__idx_k_v' keys=200 data_size=0 bytes levels=1 +test.t_a analyze Note [TIDESDB] idx 'k_v' sampled=200 distinct=10 rec_per_key=20 +test.t_a analyze status OK +ANALYZE TABLE t_b; +Table Op Msg_type Msg_text +test.t_b analyze status Engine-independent statistics collected +test.t_b analyze Note [TIDESDB] CF 'test__t_b' total_keys=75 data_size=0 bytes memtable=1725 bytes levels=1 read_amp=1.00 cache_hit=0.0% +test.t_b analyze Note [TIDESDB] avg_key=6.9 bytes avg_value=16.1 bytes +test.t_b analyze Note [TIDESDB] level 1 sstables=0 size=0 bytes keys=0 +test.t_b analyze Note [TIDESDB] WA user=1425 wal=0 flush=0 (0 ssts) compact_write=0 (0 ssts) compact_read=0 ratio=0.00x +test.t_b analyze status OK +# each table reports its own distinct row count, not the shared-memtable sum +# t_a -> 200 +SELECT TABLE_ROWS FROM information_schema.TABLES +WHERE table_schema = DATABASE() AND table_name = 't_a'; +TABLE_ROWS +200 +# t_b -> 75 +SELECT TABLE_ROWS FROM information_schema.TABLES +WHERE table_schema = DATABASE() AND table_name = 't_b'; +TABLE_ROWS +75 +# inserting more rows raises the estimate on the next refresh +ANALYZE TABLE t_a; +Table Op Msg_type Msg_text +test.t_a analyze status Engine-independent statistics collected +test.t_a analyze Note [TIDESDB] CF 'test__t_a' total_keys=300 data_size=0 bytes memtable=6900 bytes levels=1 read_amp=1.00 cache_hit=0.0% +test.t_a analyze Note [TIDESDB] avg_key=6.9 bytes avg_value=16.1 bytes +test.t_a analyze Note [TIDESDB] level 1 sstables=0 size=0 bytes keys=0 +test.t_a analyze Note [TIDESDB] WA user=5700 wal=0 flush=0 (0 ssts) compact_write=0 (0 ssts) compact_read=0 ratio=0.00x +test.t_a analyze Note [TIDESDB] idx CF 'test__t_a__idx_k_v' keys=300 data_size=0 bytes levels=1 +test.t_a analyze Note [TIDESDB] idx 'k_v' sampled=300 distinct=10 rec_per_key=30 +test.t_a analyze status OK +# t_a -> 300 +SELECT TABLE_ROWS FROM information_schema.TABLES +WHERE table_schema = DATABASE() AND table_name = 't_a'; +TABLE_ROWS +300 +DROP TABLE t_a; +DROP TABLE t_b; +# +# Done. diff --git a/mysql-test/suite/tidesdb/r/tidesdb_secondary_gt_range.result b/mysql-test/suite/tidesdb/r/tidesdb_secondary_gt_range.result new file mode 100644 index 00000000..d2589d10 --- /dev/null +++ b/mysql-test/suite/tidesdb/r/tidesdb_secondary_gt_range.result @@ -0,0 +1,78 @@ +# +# ============================================ +# > X excludes all X duplicates, includes only greater values +# ============================================ +# +CREATE TABLE t1 (id INT PRIMARY KEY, u INT, KEY k_u (u)) ENGINE=TIDESDB; +INSERT INTO t1 VALUES +(1, 2), (2, 2), (3, 2), (4, 2), (5, 2), +(6, 3), (7, 3), +(8, 4), +(9, 1), (10, 1); +# u > 2 -> only the u=3 and u=4 rows +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 2 ORDER BY u, id; +id u +6 3 +7 3 +8 4 +# u >= 2 (control) -> includes the u=2 rows +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u >= 2 ORDER BY u, id; +id u +1 2 +2 2 +3 2 +4 2 +5 2 +6 3 +7 3 +8 4 +# u > 4 (greatest value) -> empty +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 4 ORDER BY u, id; +id u +# bounded range u > 2 AND u < 4 -> only u=3 +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 2 AND u < 4 ORDER BY id; +id u +6 3 +7 3 +# +# ============================================ +# Pathological: a row whose PK encodes to the max key bytes (INT max) +# sits at X's upper bound and must still be excluded by > X +# ============================================ +# +INSERT INTO t1 VALUES (2147483647, 2); +# the max-PK row has u=2, so u > 2 must NOT return it +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 2 ORDER BY u, id; +id u +6 3 +7 3 +8 4 +# u >= 2 must include it +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u >= 2 AND u < 3 ORDER BY id; +id u +1 2 +2 2 +3 2 +4 2 +5 2 +2147483647 2 +# +# ============================================ +# Composite index: > on the leading part +# ============================================ +# +CREATE TABLE t2 (id INT PRIMARY KEY, a INT, b INT, KEY k_ab (a, b)) ENGINE=TIDESDB; +INSERT INTO t2 VALUES +(1, 1, 10), (2, 1, 20), (3, 1, 30), +(4, 2, 10), (5, 2, 20), +(6, 3, 99); +# a > 1 -> only a=2 and a=3 rows +SELECT id, a, b FROM t2 FORCE INDEX (k_ab) WHERE a > 1 ORDER BY a, b; +id a b +4 2 10 +5 2 20 +6 3 99 +DROP TABLE t1; +DROP TABLE t2; +# +# Done. diff --git a/mysql-test/suite/tidesdb/r/tidesdb_stmt_atomicity.result b/mysql-test/suite/tidesdb/r/tidesdb_stmt_atomicity.result new file mode 100644 index 00000000..129a69fb --- /dev/null +++ b/mysql-test/suite/tidesdb/r/tidesdb_stmt_atomicity.result @@ -0,0 +1,162 @@ +# +# ============================================ +# TEST 1: failed statement leaves earlier ones intact (PRIMARY KEY) +# ============================================ +# +CREATE TABLE t1 (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB; +START TRANSACTION; +INSERT INTO t1 VALUES (1, 10); +INSERT INTO t1 VALUES (2, 20); +# row 3 inserts, then row 1 duplicates -> whole statement rolls back +INSERT INTO t1 VALUES (3, 30), (1, 99); +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +# inside the txn: 1 and 2 survive, 3 is gone, 1 is unchanged (v=10) +SELECT * FROM t1 ORDER BY id; +id v +1 10 +2 20 +COMMIT; +# after commit: same result persisted +SELECT * FROM t1 ORDER BY id; +id v +1 10 +2 20 +# +# ============================================ +# TEST 2: re-insert of a rolled-back key succeeds (txn_key_state revert) +# ============================================ +# +START TRANSACTION; +INSERT INTO t1 VALUES (5, 50); +# row 6 inserts, then row 5 duplicates -> statement rolls back, 6 undone +INSERT INTO t1 VALUES (6, 60), (5, 999); +ERROR 23000: Duplicate entry '5' for key 'PRIMARY' +# 6 was rolled back, so re-inserting it must NOT report a false duplicate +INSERT INTO t1 VALUES (6, 66); +COMMIT; +SELECT * FROM t1 ORDER BY id; +id v +1 10 +2 20 +5 50 +6 66 +# +# ============================================ +# TEST 3: UNIQUE secondary index revert + re-insert +# ============================================ +# +CREATE TABLE t2 (id INT PRIMARY KEY, u INT UNIQUE, v INT) ENGINE=TIDESDB; +START TRANSACTION; +INSERT INTO t2 VALUES (1, 100, 1); +# row (2,200) inserts, then u=100 duplicates -> statement rolls back +INSERT INTO t2 VALUES (2, 200, 2), (3, 100, 3); +ERROR 23000: Duplicate entry '100' for key 'u' +# the rolled-back unique value 200 must be free to insert again +INSERT INTO t2 VALUES (2, 200, 2); +COMMIT; +SELECT * FROM t2 ORDER BY id; +id u v +1 100 1 +2 200 2 +# +# ============================================ +# TEST 4: failed UPDATE leaves earlier UPDATE intact +# ============================================ +# +START TRANSACTION; +UPDATE t1 SET v = v + 1; +# moving id 2 onto existing id 1 duplicates the PK -> statement rolls back +UPDATE t1 SET id = 1 WHERE id = 2; +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +# first UPDATE's increment survives; row 2 keeps its id +SELECT * FROM t1 ORDER BY id; +id v +1 11 +2 21 +5 51 +6 67 +COMMIT; +SELECT * FROM t1 ORDER BY id; +id v +1 11 +2 21 +5 51 +6 67 +# +# ============================================ +# TEST 5: statement rollback preserves the transaction snapshot +# ============================================ +# +CREATE TABLE t3 (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB; +INSERT INTO t3 VALUES (1, 1); +connect con1, localhost, root,,; +connection con1; +BEGIN; +# pin con1's snapshot +SELECT * FROM t3 ORDER BY id; +id v +1 1 +connection default; +INSERT INTO t3 VALUES (2, 2); +connection con1; +# a failed statement inside con1's txn must roll back only itself... +INSERT INTO t3 VALUES (3, 3), (1, 3); +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +# ...and must NOT shift the snapshot: con1 still must not see row 2 +SELECT * FROM t3 ORDER BY id; +id v +1 1 +COMMIT; +# after commit the snapshot is released, row 2 becomes visible +SELECT * FROM t3 ORDER BY id; +id v +1 1 +2 2 +disconnect con1; +connection default; +# +# ============================================ +# TEST 6: FULLTEXT index stays consistent after a statement rollback +# ============================================ +# +CREATE TABLE ft (id INT PRIMARY KEY, body TEXT, FULLTEXT(body)) ENGINE=TIDESDB; +START TRANSACTION; +INSERT INTO ft VALUES (1, 'alpha beta'); +# row 2 (gamma delta) inserts, then id 1 duplicates -> statement rolls back +INSERT INTO ft VALUES (2, 'gamma delta'), (1, 'dup'); +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +COMMIT; +# gamma was rolled back -> no match; alpha (committed row 1) still matches +SELECT id FROM ft WHERE MATCH(body) AGAINST('gamma'); +id +SELECT id FROM ft WHERE MATCH(body) AGAINST('alpha'); +id +1 +# a fresh insert of the rolled-back term indexes cleanly +INSERT INTO ft VALUES (2, 'gamma delta'); +SELECT id FROM ft WHERE MATCH(body) AGAINST('gamma'); +id +2 +# +# ============================================ +# TEST 7: user SAVEPOINT still works alongside statement savepoints +# ============================================ +# +START TRANSACTION; +INSERT INTO t1 VALUES (40, 400); +SAVEPOINT a; +INSERT INTO t1 VALUES (41, 410); +ROLLBACK TO SAVEPOINT a; +# 40 survives, 41 undone by the user rollback +INSERT INTO t1 VALUES (42, 420); +COMMIT; +SELECT id FROM t1 WHERE id >= 40 ORDER BY id; +id +40 +42 +DROP TABLE ft; +DROP TABLE t3; +DROP TABLE t2; +DROP TABLE t1; +# +# Done. diff --git a/mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.opt b/mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.opt new file mode 100644 index 00000000..5737dfca --- /dev/null +++ b/mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.opt @@ -0,0 +1,2 @@ +--plugin-load-add=file_key_management +--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt diff --git a/mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.test b/mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.test new file mode 100644 index 00000000..d726a6c6 --- /dev/null +++ b/mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.test @@ -0,0 +1,74 @@ +--source include/have_tidesdb.inc +--source include/not_embedded.inc + +# +# An encrypted table stores ciphertext in its data column family. Ciphertext +# does not compress, so the engine forces the data CF's compression off even +# when the table selects a compression algorithm, avoiding wasted CPU on every +# flush and compaction. Secondary-index CFs hold unencrypted comparable keys +# and must keep the selected compression. +# +# We verify by reading the per-CF config.ini the library writes at create time. +# The data home is the sibling "tidesdb_data" directory of the server datadir. +# + +--echo # +--echo # ============================================ +--echo # Encrypted table: data CF compression forced off, index CF keeps it +--echo # ============================================ +--echo # +CREATE TABLE enc_lz4 ( + id INT PRIMARY KEY, + u INT, + v VARCHAR(200), + KEY k_u (u) +) ENGINE=TIDESDB `ENCRYPTED`=YES COMPRESSION='LZ4'; + +--echo # the table still reports the requested COMPRESSION option to the user +SHOW CREATE TABLE enc_lz4; + +--echo # data CF: compression_algorithm = NONE (ciphertext is incompressible) +--exec grep '^compression_algorithm' $MYSQLTEST_VARDIR/mysqld.1/tidesdb_data/test__enc_lz4/config.ini +--echo # index CF: compression_algorithm = LZ4 (unencrypted keys keep compression) +--exec grep '^compression_algorithm' $MYSQLTEST_VARDIR/mysqld.1/tidesdb_data/test__enc_lz4__idx_k_u/config.ini + +--echo # +--echo # ============================================ +--echo # Control: non-encrypted table keeps the selected compression on its data CF +--echo # ============================================ +--echo # +CREATE TABLE plain_lz4 ( + id INT PRIMARY KEY, + v VARCHAR(200) +) ENGINE=TIDESDB COMPRESSION='LZ4'; + +--echo # data CF: compression_algorithm = LZ4 (not encrypted, so compression stays) +--exec grep '^compression_algorithm' $MYSQLTEST_VARDIR/mysqld.1/tidesdb_data/test__plain_lz4/config.ini + +--echo # +--echo # ============================================ +--echo # Data still round-trips under an encrypted, compression-disabled CF +--echo # ============================================ +--echo # +INSERT INTO enc_lz4 VALUES (1, 10, 'alpha'), (2, 20, 'beta'), (3, 30, 'gamma'); +UPDATE enc_lz4 SET v = 'delta' WHERE id = 2; +DELETE FROM enc_lz4 WHERE id = 3; +SELECT * FROM enc_lz4 ORDER BY id; +--echo # covering index scan on the still-compressed index CF +SELECT id FROM enc_lz4 WHERE u = 10; + +--echo # +--echo # ============================================ +--echo # TRUNCATE recreates the data CF with compression still off +--echo # ============================================ +--echo # +TRUNCATE TABLE enc_lz4; +--exec grep '^compression_algorithm' $MYSQLTEST_VARDIR/mysqld.1/tidesdb_data/test__enc_lz4/config.ini +--exec grep '^compression_algorithm' $MYSQLTEST_VARDIR/mysqld.1/tidesdb_data/test__enc_lz4__idx_k_u/config.ini + +DROP TABLE enc_lz4; +DROP TABLE plain_lz4; + +--echo # +--source include/cleanup_tidesdb.inc +--echo # Done. diff --git a/mysql-test/suite/tidesdb/t/tidesdb_keyread_decodable.test b/mysql-test/suite/tidesdb/t/tidesdb_keyread_decodable.test new file mode 100644 index 00000000..6cb50312 --- /dev/null +++ b/mysql-test/suite/tidesdb/t/tidesdb_keyread_decodable.test @@ -0,0 +1,117 @@ +--source include/have_tidesdb.inc +--source include/not_embedded.inc + +# +# HA_KEYREAD_ONLY (index-only "Using index" plans) must only be advertised for +# indexes whose columns can be reconstructed from their mem-comparable sort key +# by decode_sort_key_part. For undecodable column types (VARCHAR, DECIMAL, +# floating point, multi-byte CHAR) a keyread would fall back to a primary-key +# row fetch at runtime, so advertising a covering plan mis-prices the query. +# +# We FORCE INDEX so the plan deterministically uses the secondary index, then +# read the Extra column: "Using index" appears only when the index-only read is +# genuinely available. The rows estimate is masked as it is not stable. +# + +--echo # +--echo # ============================================ +--echo # Decodable column types: covering (Using index) IS offered +--echo # ============================================ +--echo # +CREATE TABLE cov_ok ( + id INT PRIMARY KEY, + i INT, + b BIGINT, + d DATE, + ts DATETIME, + c CHAR(8) CHARACTER SET binary, + KEY k_i (i), + KEY k_b (b), + KEY k_d (d), + KEY k_ts (ts), + KEY k_c (c) +) ENGINE=TIDESDB; +INSERT INTO cov_ok VALUES + (1, 10, 100, '2020-01-01', '2020-01-01 10:00:00', 'aaa'), + (2, 20, 200, '2021-02-02', '2021-02-02 11:00:00', 'bbb'), + (3, 30, 300, '2022-03-03', '2022-03-03 12:00:00', 'ccc'); + +--replace_column 9 # +EXPLAIN SELECT i FROM cov_ok FORCE INDEX (k_i) WHERE i = 20; +--replace_column 9 # +EXPLAIN SELECT b FROM cov_ok FORCE INDEX (k_b) WHERE b = 200; +--replace_column 9 # +EXPLAIN SELECT d FROM cov_ok FORCE INDEX (k_d) WHERE d = '2021-02-02'; +--replace_column 9 # +EXPLAIN SELECT ts FROM cov_ok FORCE INDEX (k_ts) WHERE ts = '2021-02-02 11:00:00'; +--replace_column 9 # +EXPLAIN SELECT c FROM cov_ok FORCE INDEX (k_c) WHERE c = 'bbb'; + +--echo # +--echo # ============================================ +--echo # Undecodable column types: covering is NOT offered (no Using index) +--echo # ============================================ +--echo # +CREATE TABLE cov_no ( + id INT PRIMARY KEY, + s VARCHAR(50), + u CHAR(8) CHARACTER SET utf8mb4, + m DECIMAL(10,2), + f DOUBLE, + KEY k_s (s), + KEY k_u (u), + KEY k_m (m), + KEY k_f (f) +) ENGINE=TIDESDB; +INSERT INTO cov_no VALUES + (1, 'alpha', 'x1', 1.50, 1.5), + (2, 'beta', 'x2', 2.50, 2.5), + (3, 'gamma', 'x3', 3.50, 3.5); + +--replace_column 9 # +EXPLAIN SELECT s FROM cov_no FORCE INDEX (k_s) WHERE s = 'beta'; +--replace_column 9 # +EXPLAIN SELECT u FROM cov_no FORCE INDEX (k_u) WHERE u = 'x2'; +--replace_column 9 # +EXPLAIN SELECT m FROM cov_no FORCE INDEX (k_m) WHERE m = 2.50; +--replace_column 9 # +EXPLAIN SELECT f FROM cov_no FORCE INDEX (k_f) WHERE f = 2.5; + +--echo # results are still correct on the non-covering (PK fetch) path +SELECT s FROM cov_no FORCE INDEX (k_s) WHERE s = 'beta'; +SELECT m FROM cov_no FORCE INDEX (k_m) WHERE m = 2.50; + +--echo # +--echo # ============================================ +--echo # Composite index: covering decided per key part, not per whole index +--echo # ============================================ +--echo # +CREATE TABLE cov_mix ( + id INT PRIMARY KEY, + a INT, + b INT, + s VARCHAR(50), + KEY k_ab (a, b), + KEY k_as (a, s) +) ENGINE=TIDESDB; +INSERT INTO cov_mix VALUES (1, 1, 10, 'x'), (2, 2, 20, 'y'), (3, 3, 30, 'z'); + +--echo # (a INT, b INT): both decodable -> Using index +--replace_column 9 # +EXPLAIN SELECT a, b FROM cov_mix FORCE INDEX (k_ab) WHERE a = 2; +--echo # (a INT, s VARCHAR), reading only the decodable a -> still Using index +--replace_column 9 # +EXPLAIN SELECT a FROM cov_mix FORCE INDEX (k_as) WHERE a = 2; +--echo # (a INT, s VARCHAR), reading the undecodable s -> no Using index +--replace_column 9 # +EXPLAIN SELECT a, s FROM cov_mix FORCE INDEX (k_as) WHERE a = 2; +--echo # result on the s (non-covering) path is still correct +SELECT a, s FROM cov_mix FORCE INDEX (k_as) WHERE a = 2; + +DROP TABLE cov_ok; +DROP TABLE cov_no; +DROP TABLE cov_mix; + +--echo # +--source include/cleanup_tidesdb.inc +--echo # Done. diff --git a/mysql-test/suite/tidesdb/t/tidesdb_row_cardinality.test b/mysql-test/suite/tidesdb/t/tidesdb_row_cardinality.test new file mode 100644 index 00000000..d4290324 --- /dev/null +++ b/mysql-test/suite/tidesdb/t/tidesdb_row_cardinality.test @@ -0,0 +1,62 @@ +--source include/have_tidesdb.inc +--source include/not_embedded.inc + +# +# The row-count estimate (information_schema.TABLES.TABLE_ROWS, stats.records) +# is derived from tidesdb_cf_estimate_cardinality, which counts each key once +# per SSTable regardless of how many MVCC versions it holds and also counts the +# CF's entries in the (default) shared unified memtable. For a table written +# with only distinct keys and no updates every key exists exactly once, so the +# estimate is exact. Two independent tables confirm the count is attributed to +# each column family separately rather than summing the shared memtable. +# + +CREATE TABLE t_a (id INT PRIMARY KEY, v INT, KEY k_v (v)) ENGINE=TIDESDB; +CREATE TABLE t_b (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB; + +--disable_query_log +let $i = 1; +while ($i <= 200) +{ + eval INSERT INTO t_a VALUES ($i, $i MOD 10); + inc $i; +} +let $i = 1; +while ($i <= 75) +{ + eval INSERT INTO t_b VALUES ($i, $i); + inc $i; +} +--enable_query_log + +ANALYZE TABLE t_a; +ANALYZE TABLE t_b; + +--echo # each table reports its own distinct row count, not the shared-memtable sum +--echo # t_a -> 200 +SELECT TABLE_ROWS FROM information_schema.TABLES + WHERE table_schema = DATABASE() AND table_name = 't_a'; +--echo # t_b -> 75 +SELECT TABLE_ROWS FROM information_schema.TABLES + WHERE table_schema = DATABASE() AND table_name = 't_b'; + +--echo # inserting more rows raises the estimate on the next refresh +--disable_query_log +let $i = 201; +while ($i <= 300) +{ + eval INSERT INTO t_a VALUES ($i, $i MOD 10); + inc $i; +} +--enable_query_log +ANALYZE TABLE t_a; +--echo # t_a -> 300 +SELECT TABLE_ROWS FROM information_schema.TABLES + WHERE table_schema = DATABASE() AND table_name = 't_a'; + +DROP TABLE t_a; +DROP TABLE t_b; + +--echo # +--source include/cleanup_tidesdb.inc +--echo # Done. diff --git a/mysql-test/suite/tidesdb/t/tidesdb_secondary_gt_range.test b/mysql-test/suite/tidesdb/t/tidesdb_secondary_gt_range.test new file mode 100644 index 00000000..3aa7c0bf --- /dev/null +++ b/mysql-test/suite/tidesdb/t/tidesdb_secondary_gt_range.test @@ -0,0 +1,65 @@ +--source include/have_tidesdb.inc +--source include/not_embedded.inc + +# +# WHERE seccol > X on a secondary index (HA_READ_AFTER_KEY) must skip every +# entry equal to X and position on the first strictly-greater value. The engine +# now does this with a single upper-bound seek instead of stepping over each +# duplicate. These tests pin the behaviour, including the low-cardinality case +# (many rows share X) and the pathological row whose primary key encodes to the +# maximum key bytes and lands exactly on the upper bound. +# + +--echo # +--echo # ============================================ +--echo # > X excludes all X duplicates, includes only greater values +--echo # ============================================ +--echo # +CREATE TABLE t1 (id INT PRIMARY KEY, u INT, KEY k_u (u)) ENGINE=TIDESDB; +INSERT INTO t1 VALUES + (1, 2), (2, 2), (3, 2), (4, 2), (5, 2), + (6, 3), (7, 3), + (8, 4), + (9, 1), (10, 1); + +--echo # u > 2 -> only the u=3 and u=4 rows +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 2 ORDER BY u, id; +--echo # u >= 2 (control) -> includes the u=2 rows +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u >= 2 ORDER BY u, id; +--echo # u > 4 (greatest value) -> empty +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 4 ORDER BY u, id; +--echo # bounded range u > 2 AND u < 4 -> only u=3 +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 2 AND u < 4 ORDER BY id; + +--echo # +--echo # ============================================ +--echo # Pathological: a row whose PK encodes to the max key bytes (INT max) +--echo # sits at X's upper bound and must still be excluded by > X +--echo # ============================================ +--echo # +INSERT INTO t1 VALUES (2147483647, 2); +--echo # the max-PK row has u=2, so u > 2 must NOT return it +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 2 ORDER BY u, id; +--echo # u >= 2 must include it +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u >= 2 AND u < 3 ORDER BY id; + +--echo # +--echo # ============================================ +--echo # Composite index: > on the leading part +--echo # ============================================ +--echo # +CREATE TABLE t2 (id INT PRIMARY KEY, a INT, b INT, KEY k_ab (a, b)) ENGINE=TIDESDB; +INSERT INTO t2 VALUES + (1, 1, 10), (2, 1, 20), (3, 1, 30), + (4, 2, 10), (5, 2, 20), + (6, 3, 99); + +--echo # a > 1 -> only a=2 and a=3 rows +SELECT id, a, b FROM t2 FORCE INDEX (k_ab) WHERE a > 1 ORDER BY a, b; + +DROP TABLE t1; +DROP TABLE t2; + +--echo # +--source include/cleanup_tidesdb.inc +--echo # Done. diff --git a/mysql-test/suite/tidesdb/t/tidesdb_stmt_atomicity.test b/mysql-test/suite/tidesdb/t/tidesdb_stmt_atomicity.test new file mode 100644 index 00000000..8a7268c8 --- /dev/null +++ b/mysql-test/suite/tidesdb/t/tidesdb_stmt_atomicity.test @@ -0,0 +1,152 @@ +--source include/have_tidesdb.inc +--source include/not_embedded.inc + +# +# Statement atomicity inside a multi-statement transaction. +# A statement that fails part-way (e.g. a multi-row INSERT whose later row +# duplicates a key) must roll back ONLY that statement, leaving earlier +# statements in the same BEGIN...COMMIT intact, and must not shift the +# transaction snapshot. Exercises the per-statement savepoint plus the +# revert of plugin-side shadow state (txn_key_state, unique-secondary and +# FTS bookkeeping). +# + +--echo # +--echo # ============================================ +--echo # TEST 1: failed statement leaves earlier ones intact (PRIMARY KEY) +--echo # ============================================ +--echo # +CREATE TABLE t1 (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB; + +START TRANSACTION; +INSERT INTO t1 VALUES (1, 10); +INSERT INTO t1 VALUES (2, 20); +--echo # row 3 inserts, then row 1 duplicates -> whole statement rolls back +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES (3, 30), (1, 99); +--echo # inside the txn: 1 and 2 survive, 3 is gone, 1 is unchanged (v=10) +SELECT * FROM t1 ORDER BY id; +COMMIT; +--echo # after commit: same result persisted +SELECT * FROM t1 ORDER BY id; + +--echo # +--echo # ============================================ +--echo # TEST 2: re-insert of a rolled-back key succeeds (txn_key_state revert) +--echo # ============================================ +--echo # +START TRANSACTION; +INSERT INTO t1 VALUES (5, 50); +--echo # row 6 inserts, then row 5 duplicates -> statement rolls back, 6 undone +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES (6, 60), (5, 999); +--echo # 6 was rolled back, so re-inserting it must NOT report a false duplicate +INSERT INTO t1 VALUES (6, 66); +COMMIT; +SELECT * FROM t1 ORDER BY id; + +--echo # +--echo # ============================================ +--echo # TEST 3: UNIQUE secondary index revert + re-insert +--echo # ============================================ +--echo # +CREATE TABLE t2 (id INT PRIMARY KEY, u INT UNIQUE, v INT) ENGINE=TIDESDB; + +START TRANSACTION; +INSERT INTO t2 VALUES (1, 100, 1); +--echo # row (2,200) inserts, then u=100 duplicates -> statement rolls back +--error ER_DUP_ENTRY +INSERT INTO t2 VALUES (2, 200, 2), (3, 100, 3); +--echo # the rolled-back unique value 200 must be free to insert again +INSERT INTO t2 VALUES (2, 200, 2); +COMMIT; +SELECT * FROM t2 ORDER BY id; + +--echo # +--echo # ============================================ +--echo # TEST 4: failed UPDATE leaves earlier UPDATE intact +--echo # ============================================ +--echo # +START TRANSACTION; +UPDATE t1 SET v = v + 1; +--echo # moving id 2 onto existing id 1 duplicates the PK -> statement rolls back +--error ER_DUP_ENTRY +UPDATE t1 SET id = 1 WHERE id = 2; +--echo # first UPDATE's increment survives; row 2 keeps its id +SELECT * FROM t1 ORDER BY id; +COMMIT; +SELECT * FROM t1 ORDER BY id; + +--echo # +--echo # ============================================ +--echo # TEST 5: statement rollback preserves the transaction snapshot +--echo # ============================================ +--echo # +CREATE TABLE t3 (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB; +INSERT INTO t3 VALUES (1, 1); + +connect (con1, localhost, root,,); +connection con1; +BEGIN; +--echo # pin con1's snapshot +SELECT * FROM t3 ORDER BY id; + +connection default; +INSERT INTO t3 VALUES (2, 2); + +connection con1; +--echo # a failed statement inside con1's txn must roll back only itself... +--error ER_DUP_ENTRY +INSERT INTO t3 VALUES (3, 3), (1, 3); +--echo # ...and must NOT shift the snapshot: con1 still must not see row 2 +SELECT * FROM t3 ORDER BY id; +COMMIT; +--echo # after commit the snapshot is released, row 2 becomes visible +SELECT * FROM t3 ORDER BY id; + +disconnect con1; +connection default; + +--echo # +--echo # ============================================ +--echo # TEST 6: FULLTEXT index stays consistent after a statement rollback +--echo # ============================================ +--echo # +CREATE TABLE ft (id INT PRIMARY KEY, body TEXT, FULLTEXT(body)) ENGINE=TIDESDB; + +START TRANSACTION; +INSERT INTO ft VALUES (1, 'alpha beta'); +--echo # row 2 (gamma delta) inserts, then id 1 duplicates -> statement rolls back +--error ER_DUP_ENTRY +INSERT INTO ft VALUES (2, 'gamma delta'), (1, 'dup'); +COMMIT; +--echo # gamma was rolled back -> no match; alpha (committed row 1) still matches +SELECT id FROM ft WHERE MATCH(body) AGAINST('gamma'); +SELECT id FROM ft WHERE MATCH(body) AGAINST('alpha'); +--echo # a fresh insert of the rolled-back term indexes cleanly +INSERT INTO ft VALUES (2, 'gamma delta'); +SELECT id FROM ft WHERE MATCH(body) AGAINST('gamma'); + +--echo # +--echo # ============================================ +--echo # TEST 7: user SAVEPOINT still works alongside statement savepoints +--echo # ============================================ +--echo # +START TRANSACTION; +INSERT INTO t1 VALUES (40, 400); +SAVEPOINT a; +INSERT INTO t1 VALUES (41, 410); +ROLLBACK TO SAVEPOINT a; +--echo # 40 survives, 41 undone by the user rollback +INSERT INTO t1 VALUES (42, 420); +COMMIT; +SELECT id FROM t1 WHERE id >= 40 ORDER BY id; + +DROP TABLE ft; +DROP TABLE t3; +DROP TABLE t2; +DROP TABLE t1; + +--echo # +--source include/cleanup_tidesdb.inc +--echo # Done. diff --git a/tidesdb/ha_tidesdb.cc b/tidesdb/ha_tidesdb.cc index 1b8dc3cf..aef55aa1 100644 --- a/tidesdb/ha_tidesdb.cc +++ b/tidesdb/ha_tidesdb.cc @@ -997,6 +997,40 @@ static int row_lock_acquire(tidesdb_trx_t *trx, const uchar *key, uint len, THD promotes any waiters now compatible with the remaining granted set, and broadcasts the lock's cond. Called from commit and rollback. */ +/* Release one granted request, promoting and waking any waiters it unblocks. + Caller owns removing req from its trx->held_locks_head chain. */ +static void tdb_lock_release_one(tdb_lock_request_t *req) +{ + tdb_row_lock_t *lock = req->lock; + uint part_idx = lock->partition; + tdb_lock_partition_t *part = &lock_partitions[part_idx]; + + mysql_mutex_lock(&part->mutex); + + /* Unlink req from lock->granted_head. */ + tdb_lock_request_t **pp = &lock->granted_head; + while (*pp && *pp != req) pp = &(*pp)->list_next; + if (*pp == req) *pp = req->list_next; + + /* Promote any waiters now grantable, then wake them up. */ + bool had_waiters = (lock->waiting_head != NULL); + tdb_lock_promote_waiters(lock); + bool promoted_any = had_waiters && (lock->granted_head != NULL); + + /* If nothing references this slot any more, unlink it from the + hash chain and stash it on the partition freelist so the next + acquire can reuse it without growing the chain. Slot memory + is retained across reuse for the deadlock walker. */ + tdb_lock_freelist_if_empty(part, lock); + + mysql_mutex_unlock(&part->mutex); + + if (had_waiters && (promoted_any || lock->waiting_head == NULL)) + mysql_cond_broadcast(&lock->cond); + + my_free(req); +} + static void row_locks_release_all(tidesdb_trx_t *trx) { if (!lock_partitions || !trx) return; @@ -1006,40 +1040,134 @@ static void row_locks_release_all(tidesdb_trx_t *trx) while (req) { tdb_lock_request_t *next = req->held_next; - tdb_row_lock_t *lock = req->lock; - uint part_idx = lock->partition; - tdb_lock_partition_t *part = &lock_partitions[part_idx]; + tdb_lock_release_one(req); + released++; + req = next; + } + trx->held_locks_head = NULL; + trx->waiting_on_lock.store(NULL, std::memory_order_relaxed); + if (released > 0) srv_stat_lock_held.fetch_sub(released, std::memory_order_relaxed); +} - mysql_mutex_lock(&part->mutex); +/* Release only the locks acquired since a statement-start marker, leaving the + marker and everything below it (locks from earlier statements) held. Because + row_lock_acquire pushes new grants onto the head and re-entry never adds a + duplicate request, the statement's locks are exactly the run from the current + head down to (but excluding) marker. */ +static void row_locks_release_since(tidesdb_trx_t *trx, tdb_lock_request_t *marker) +{ + if (!lock_partitions || !trx) return; - /* Unlink req from lock->granted_head. */ - tdb_lock_request_t **pp = &lock->granted_head; - while (*pp && *pp != req) pp = &(*pp)->list_next; - if (*pp == req) *pp = req->list_next; + long long released = 0; + tdb_lock_request_t *req = trx->held_locks_head; + while (req && req != marker) + { + tdb_lock_request_t *next = req->held_next; + tdb_lock_release_one(req); + released++; + req = next; + } + trx->held_locks_head = marker; + if (released > 0) srv_stat_lock_held.fetch_sub(released, std::memory_order_relaxed); +} - /* Promote any waiters now grantable, then wake them up. */ - bool had_waiters = (lock->waiting_head != NULL); - tdb_lock_promote_waiters(lock); - bool promoted_any = had_waiters && (lock->granted_head != NULL); +/* Reserved name of the per-statement savepoint. The SQL SAVEPOINT callbacks + synthesize "sv_%p" names, so this never collides with a user savepoint. */ +static constexpr const char TIDESDB_STMT_SAVEPOINT[] = "stmt"; - /* If nothing references this slot any more, unlink it from the - hash chain and stash it on the partition freelist so the next - acquire can reuse it without growing the chain. Slot memory - is retained across reuse for the deadlock walker. */ - tdb_lock_freelist_if_empty(part, lock); +/* Set trx->txn_key_state[key] = val, journaling the prior value while a + statement savepoint is armed so a statement rollback can restore it. */ +static inline void trx_set_key_state(tidesdb_trx_t *trx, const std::string &key, bool val) +{ + if (trx->stmt_savepoint_active) + { + auto it = trx->txn_key_state.find(key); + signed char prior = (it == trx->txn_key_state.end()) ? -1 : (it->second ? 1 : 0); + trx->stmt_key_state_undo.push_back({key, prior}); + } + trx->txn_key_state[key] = val; +} - mysql_mutex_unlock(&part->mutex); +/* Arm a statement savepoint at statement start. Caller guarantees we are in a + multi-statement transaction (not autocommit/DDL), trx->txn exists, and no + statement savepoint is currently armed. On any failure we leave the savepoint + unarmed so a later statement rollback safely falls back to full rollback. */ +static void stmt_savepoint_arm(tidesdb_trx_t *trx) +{ + /* Defensive: a prior statement's savepoint should already be gone (released + on success, removed on rollback), but a user ROLLBACK TO SAVEPOINT can + collaterally drop it; releasing again is a cheap no-op if absent. */ + (void)tidesdb_txn_release_savepoint(trx->txn, TIDESDB_STMT_SAVEPOINT); + if (tidesdb_txn_savepoint(trx->txn, TIDESDB_STMT_SAVEPOINT) != TDB_SUCCESS) return; - if (had_waiters && (promoted_any || lock->waiting_head == NULL)) - mysql_cond_broadcast(&lock->cond); + trx->stmt_savepoint_active = true; + trx->stmt_lock_marker = trx->held_locks_head; + trx->stmt_key_state_undo.clear(); + trx->stmt_fts_snapshot = trx->fts_meta_pending; + trx->stmt_fts_dirty_snapshot = trx->fts_meta_dirty; +} - my_free(req); - released++; - req = next; +/* Statement completed successfully -- the statement's writes are now permanent + within the (still uncommitted) txn. Drop the savepoint and per-statement undo + state so the next statement re-arms cleanly. */ +static void stmt_savepoint_disarm(tidesdb_trx_t *trx) +{ + if (!trx->stmt_savepoint_active) return; + (void)tidesdb_txn_release_savepoint(trx->txn, TIDESDB_STMT_SAVEPOINT); + trx->stmt_savepoint_active = false; + trx->stmt_lock_marker = nullptr; + trx->stmt_key_state_undo.clear(); + trx->stmt_fts_snapshot.clear(); +} + +/* Roll back just the current statement's effects to the armed savepoint. + Returns true if the partial rollback was performed, false if the caller must + fall back to a full transaction rollback (no savepoint armed, or the library + savepoint is gone because a bulk mid-commit reset the txn or a user ROLLBACK + TO SAVEPOINT removed it). */ +static bool stmt_savepoint_rollback(tidesdb_trx_t *trx) +{ + if (!trx->stmt_savepoint_active) return false; + + int rc = tidesdb_txn_rollback_to_savepoint(trx->txn, TIDESDB_STMT_SAVEPOINT); + if (rc != TDB_SUCCESS) + { + /* Savepoint vanished under us -- cannot do a partial rollback. */ + trx->stmt_savepoint_active = false; + trx->stmt_lock_marker = nullptr; + trx->stmt_key_state_undo.clear(); + trx->stmt_fts_snapshot.clear(); + return false; } - trx->held_locks_head = NULL; - trx->waiting_on_lock.store(NULL, std::memory_order_relaxed); - if (released > 0) srv_stat_lock_held.fetch_sub(released, std::memory_order_relaxed); + + /* Revert the plugin-side shadow state to the same boundary. Undo the + txn_key_state mutations in reverse so a key set twice restores correctly. */ + for (auto it = trx->stmt_key_state_undo.rbegin(); it != trx->stmt_key_state_undo.rend(); ++it) + { + if (it->second < 0) + trx->txn_key_state.erase(it->first); + else + trx->txn_key_state[it->first] = (it->second == 1); + } + trx->stmt_key_state_undo.clear(); + + trx->fts_meta_pending = trx->stmt_fts_snapshot; + trx->fts_meta_dirty = trx->stmt_fts_dirty_snapshot; + trx->stmt_fts_snapshot.clear(); + + /* Drop only the locks the failed statement took; earlier statements keep + theirs. An S->X upgrade on an earlier statement's lock stays X, which is + conservative but never incorrect. */ + row_locks_release_since(trx, trx->stmt_lock_marker); + trx->stmt_lock_marker = nullptr; + + /* The library freed the txn ops appended after the savepoint, so any cached + iterator over this txn (scan_iter, dup_iter_cache_ on every handler on + this connection) now has a stale view. Bumping the generation makes each + handler rebuild lazily, exactly as a bulk mid-commit does. */ + trx->txn_generation++; + trx->stmt_savepoint_active = false; + return true; } /* Pick the lock mode for a row materialised on a read path, or report @@ -3528,6 +3656,21 @@ static tidesdb_column_family_config_t build_cf_config(const ha_table_option_stru return cfg; } +/* Rows in a table's data column family are encrypted in serialize_row before + they reach the library, and ciphertext does not compress, so running the + block and value-log compressor over it on every flush and compaction spends + CPU for no space saving. Return a copy of the config with compression forced + off when the table is encrypted, leaving every other setting untouched. This + is for the data CF only -- secondary-index CFs hold unencrypted comparable + keys and keep whatever compression the table selected. */ +static tidesdb_column_family_config_t data_cf_config(const tidesdb_column_family_config_t &cfg, + bool encrypted) +{ + tidesdb_column_family_config_t data = cfg; + if (encrypted) data.compression_algorithm = (compression_algorithm)TDB_COMPRESS_NONE; + return data; +} + /* Resolve a secondary index CF by name. Returns the CF pointer (may be NULL if not found). @@ -3755,21 +3898,13 @@ static int tidesdb_commit(handlerton *, THD *thd, bool all) { /* Statement-level commit inside a multi-statement transaction. Defer the actual commit -- writes stay buffered in the txn, - avoiding expensive txn_begin + commit per statement. - - tidesdb_txn_savepoint() deep-copies the entire - write-set (malloc+memcpy for every key/value). For a txn - with N ops across S statements, total copy cost is - O(S * N * avg_kv_size) -- quadratic and devastating for - multi-statement OLTP transactions. - - We skip the per-statement savepoint entirely. This means - statement-level rollback inside BEGIN...COMMIT falls back to - full transaction rollback (same as many simple SE's). - The trade-off is a statement failure aborts the entire txn - instead of undoing just that statement. For OLTP this is - acceptable since the client will retry the whole transaction - anyway after a conflict/error. */ + avoiding an expensive txn_begin + commit per statement. The + statement succeeded, so its writes become permanent within the + still-open txn; drop the statement savepoint armed in external_lock + so the next statement re-arms at the new boundary. (tidesdb_txn + savepoints are O(1) -- they record op/cf counts, they do not copy + the write-set -- so this per-statement arm/disarm is cheap.) */ + stmt_savepoint_disarm(trx); return 0; } @@ -3779,12 +3914,10 @@ static int tidesdb_commit(handlerton *, THD *thd, bool all) trx->txn_key_state.clear(); /* We must release any active statement savepoint before final commit/rollback. - Savepoints must be explicitly released before txn_commit. */ - if (trx->stmt_savepoint_active) - { - tidesdb_txn_release_savepoint(trx->txn, "stmt"); - trx->stmt_savepoint_active = false; - } + Savepoints must be explicitly released before txn_commit. Disarm also + clears the per-statement undo journal and fts snapshot; it leaves + fts_meta_pending intact for the flush below. */ + stmt_savepoint_disarm(trx); /* Real commit -- flush to storage. After a successful commit, we keep the txn object alive and let @@ -3867,15 +4000,25 @@ static int tidesdb_rollback(handlerton *, THD *thd, bool all) if (!is_real_rollback) { /* Statement-level rollback inside a multi-statement transaction. - Without per-statement savepoints (see tidesdb_commit note), - we fall through to full transaction rollback. This is the - same behavior as many simple storage engines and is correct -- - OLTP clients retry the entire transaction after any error. */ + Roll back only this statement's effects to the savepoint armed in + external_lock, preserving the rest of the transaction and its + snapshot. stmt_savepoint_rollback reverts the library op array and + the plugin-side shadow state (txn_key_state, fts_meta_pending, and + the statement's row locks) together. It returns false when a partial + rollback is impossible -- no savepoint armed, or a bulk mid-commit or + user ROLLBACK TO SAVEPOINT removed it -- and we fall through to a full + transaction rollback in that case. */ + if (stmt_savepoint_rollback(trx)) + { + /* Leave trx->dirty as-is: earlier statements' writes remain in the + still-open txn, and its snapshot is deliberately preserved. */ + return 0; + } } if (trx->stmt_savepoint_active) { - tidesdb_txn_release_savepoint(trx->txn, "stmt"); + tidesdb_txn_release_savepoint(trx->txn, TIDESDB_STMT_SAVEPOINT); trx->stmt_savepoint_active = false; } @@ -3884,6 +4027,9 @@ static int tidesdb_rollback(handlerton *, THD *thd, bool all) trx->fts_meta_pending.clear(); trx->fts_meta_dirty = false; trx->txn_key_state.clear(); + trx->stmt_key_state_undo.clear(); + trx->stmt_fts_snapshot.clear(); + trx->stmt_lock_marker = nullptr; /* Full rollback -- we keep txn alive for reuse via reset on next use. */ tidesdb_txn_rollback(trx->txn); @@ -3910,7 +4056,6 @@ static int tidesdb_close_connection(handlerton *, THD *thd) tidesdb_txn_rollback(trx->txn); tidesdb_txn_free(trx->txn); } - if (trx->dup_rtxn) tidesdb_txn_free(trx->dup_rtxn); delete trx; thd_set_ha_data(thd, tidesdb_hton, NULL); } @@ -5567,8 +5712,64 @@ bool ha_tidesdb::decode_int_sort_key(const uint8_t *src, uint sort_len, bool is_ Returns true on success, false for unsupported types. */ + +/* Whether decode_sort_key_part can reconstruct this field's value from its + comparable sort key. Depends only on the field type and charset, not on any + stored bytes, so index_flags can use it to decide index-only capability + without touching data. This is the single source of truth for the set of + decodable types -- decode_sort_key_part checks it first, so the two never + drift. The mem-comparable sort key is only invertible for these types; for + VARCHAR, DECIMAL, floating point, multi-byte CHAR, BLOB, ENUM/SET and the + like the sort weights are lossy, so a keyread of such a column must fall back + to the primary-key row fetch. */ +static bool is_keyread_decodable_type(const Field *f) +{ + switch (f->real_type()) + { + case MYSQL_TYPE_TINY: + case MYSQL_TYPE_SHORT: + case MYSQL_TYPE_INT24: + case MYSQL_TYPE_LONG: + case MYSQL_TYPE_LONGLONG: + case MYSQL_TYPE_YEAR: + case MYSQL_TYPE_DATE: + case MYSQL_TYPE_NEWDATE: + case MYSQL_TYPE_DATETIME: + case MYSQL_TYPE_DATETIME2: + case MYSQL_TYPE_TIMESTAMP: + case MYSQL_TYPE_TIMESTAMP2: + return true; + case MYSQL_TYPE_STRING: + /* Fixed CHAR/BINARY only for charsets whose sort weights equal the + stored bytes. */ + return f->charset() == &my_charset_bin || f->charset() == &my_charset_latin1; + default: + return false; + } +} + +/* Whether the field at key part `part` of index `keyno` reconstructs from its + sort key. Resolved through key_part.fieldnr and table_share->field rather + than key_part.field, because index_flags builds field->part_of_key on a + fresh, unopened handler at frm-parse time where key_part.field and the plugin + share are not yet available. An unresolvable field is treated conservatively + as undecodable. */ +static bool index_part_is_decodable(const TABLE_SHARE *ts, uint keyno, uint part) +{ + if (!ts || !ts->field || keyno >= ts->keys) return false; + const KEY *k = &ts->key_info[keyno]; + if (part >= k->user_defined_key_parts) return false; + uint fnr = k->key_part[part].fieldnr; + if (fnr == 0 || fnr - 1 >= ts->fields) return false; + return is_keyread_decodable_type(ts->field[fnr - 1]); +} + bool ha_tidesdb::decode_sort_key_part(const uint8_t *src, uint sort_len, Field *f, uchar *buf) { + /* Reject undecodable types up front so this stays in lockstep with + is_keyread_decodable_type, which index_flags relies on. */ + if (!is_keyread_decodable_type(f)) return false; + /* Compute the destination pointer exactly once per call. Every branch below wrote `buf + (f->ptr - f->table->record[0])` independently. */ uchar *to = buf + (uintptr_t)(f->ptr - f->table->record[0]); @@ -6146,11 +6347,12 @@ int ha_tidesdb::create(const char *name, TABLE *table_arg, HA_CREATE_INFO *creat } tidesdb_column_family_config_t cfg = build_cf_config(opts); + tidesdb_column_family_config_t data_cfg = data_cf_config(cfg, opts && opts->encrypted); /* We create main data CF (we simply skip if it already exists, e.g. crash recovery) */ if (!tidesdb_get_column_family(tdb_global, cf_name.c_str())) { - int rc = tidesdb_create_column_family(tdb_global, cf_name.c_str(), &cfg); + int rc = tidesdb_create_column_family(tdb_global, cf_name.c_str(), &data_cfg); if (rc != TDB_SUCCESS) { sql_print_error("[TIDESDB] Failed to create CF '%s' (err=%d)", cf_name.c_str(), rc); @@ -6709,21 +6911,20 @@ int ha_tidesdb::iter_read_current(uchar *buf) /* Probe whether a PK already exists for an INSERT/UPDATE uniqueness check - WITHOUT recording the read in the write transaction. - - A tracking tidesdb_txn_get() of an absent key records read-seq 0 in the txn - read-set; the library's first-committer-wins reservation then uses that 0 as - the write's conflict base, so a stale reservation slot left by any prior write - of the same key (slots survive across DROP/recreate) trips a spurious - TDB_ERR_CONFLICT under load. The probe must therefore stay out of the write - txn's read-set. We answer from two sources: - - -- same-txn pending state (trx->txn_key_state) -- this txn may have already - inserted (-> duplicate) or deleted (-> not a duplicate) the key, which - committed data alone cannot tell us; and - -- committed data, read on a dedicated READ_COMMITTED txn that is reset per - probe. Being read-only it never reserves writes (no base pollution), and - READ_COMMITTED is skipped by the snapshot floor so it never pins min_snap. + without recording the read into the write transaction's conflict footprint. + + tidesdb_txn_contains resolves at the write txn's snapshot but does not track + the read, so an absent-key probe cannot seed read-seq 0 into the first- + committer-wins reservation base -- a tracking get would, and a stale + reservation slot left by a prior write of the same key would then trip a + spurious TDB_ERR_CONFLICT under load. Uniqueness against a writer that + commits the same key after our snapshot is still enforced by this insert's + own reservation at commit time. We answer from two sources: + + -- same-txn pending state (trx->txn_key_state), which records a key this txn + has already inserted (-> duplicate) or deleted (-> not a duplicate), and + which committed data alone cannot reveal; and + -- a non-tracking existence check on the write txn for everything else. */ int ha_tidesdb::probe_pk_exists(tidesdb_trx_t *trx, const std::string &tkey, const uchar *dk, uint dk_len) @@ -6732,48 +6933,13 @@ int ha_tidesdb::probe_pk_exists(tidesdb_trx_t *trx, const std::string &tkey, con { auto it = trx->txn_key_state.find(tkey); if (it != trx->txn_key_state.end()) return it->second ? 1 : 0; - - int rc; - if (!trx->dup_rtxn) - { - rc = tidesdb_txn_begin_with_isolation(tdb_global, TDB_ISOLATION_READ_COMMITTED, - &trx->dup_rtxn); - } - else - { - /* tidesdb_txn_reset requires a committed txn (it is the post-commit - reuse path). Finalize the previous probe's read-only txn first -- - num_ops==0 at READ_COMMITTED takes the library's read-only commit - fast path, so this is cheap and never reserves anything. */ - tidesdb_txn_commit(trx->dup_rtxn); - rc = tidesdb_txn_reset(trx->dup_rtxn, TDB_ISOLATION_READ_COMMITTED); - } - if (rc != TDB_SUCCESS) return -tdb_rc_to_ha(rc, "probe_pk_exists txn"); - - uint8_t *v = NULL; - size_t l = 0; - int g = tidesdb_txn_get(trx->dup_rtxn, share->cf, dk, dk_len, &v, &l); - if (g == TDB_SUCCESS) - { - if (v) tidesdb_free(v); - return 1; - } - if (g == TDB_ERR_NOT_FOUND) return 0; - return -tdb_rc_to_ha(g, "probe_pk_exists get"); } - /* No per-connection trx (rare). Fall back to the tracking get; without a - reused write txn there is no reservation base to pollute. */ - uint8_t *v = NULL; - size_t l = 0; - int g = tidesdb_txn_get(stmt_txn, share->cf, dk, dk_len, &v, &l); - if (g == TDB_SUCCESS) - { - if (v) tidesdb_free(v); - return 1; - } + tidesdb_txn_t *probe_txn = (trx && trx->txn) ? trx->txn : stmt_txn; + int g = tidesdb_txn_contains(probe_txn, share->cf, dk, dk_len); + if (g == TDB_SUCCESS) return 1; if (g == TDB_ERR_NOT_FOUND) return 0; - return -tdb_rc_to_ha(g, "probe_pk_exists get(fallback)"); + return -tdb_rc_to_ha(g, "probe_pk_exists contains"); } /* ******************** write_row (INSERT) ******************** */ @@ -6919,7 +7085,7 @@ int ha_tidesdb::write_row(const uchar *buf) /* Not a duplicate -- record the pending insert so a later insert of the same key in this same txn is still caught (committed data cannot show a pending insert). The put below stays blind, so base = snapshot. */ - if (trx) trx->txn_key_state[tkey] = true; + if (trx) trx_set_key_state(trx, tkey, true); } /* We check UNIQUE secondary index uniqueness. This honours the @@ -7566,15 +7732,26 @@ int ha_tidesdb::index_read_map(uchar *buf, const uchar *key, key_part_map keypar } else if (find_flag == HA_READ_AFTER_KEY) { - /* We seek, then skip past any exact prefix matches */ - tidesdb_iter_seek(scan_iter, comp_key, comp_len); - while (tidesdb_iter_valid(scan_iter)) + /* Skip every entry sharing this index-column prefix in one seek + instead of stepping over them one at a time. Secondary entries + are [comparable idx cols][pk], so the greatest entry with this + prefix is comp_key followed by the maximum pk (all 0xFF), the same + upper bound the PREV branch below builds. Seeking to it lands on + the first strictly-greater index value -- the only prefix match it + can land on is the pathological row whose pk itself encodes to all + 0xFF, which we then step past. */ + uchar upper[SEC_IDX_KEY_BUF_LEN]; + memcpy(upper, comp_key, comp_len); + memset(upper + comp_len, KEY_INF_HI_BYTE, share->pk_key_len); + uint upper_len = comp_len + share->pk_key_len; + tidesdb_iter_seek(scan_iter, upper, upper_len); + if (tidesdb_iter_valid(scan_iter)) { uint8_t *ik = NULL; size_t iks = 0; - if (tidesdb_iter_key(scan_iter, &ik, &iks) != TDB_SUCCESS) break; - if (iks < comp_len || memcmp(ik, comp_key, comp_len) != 0) break; - tidesdb_iter_next(scan_iter); + if (tidesdb_iter_key(scan_iter, &ik, &iks) == TDB_SUCCESS && iks >= comp_len && + memcmp(ik, comp_key, comp_len) == 0) + tidesdb_iter_next(scan_iter); } } else if (find_flag == HA_READ_KEY_OR_PREV || find_flag == HA_READ_BEFORE_KEY || @@ -8106,14 +8283,20 @@ int ha_tidesdb::update_row(const uchar *old_data, const uchar *new_data) uint old_dk_chk_len = build_data_key(old_pk, old_pk_len, old_dk_chk); std::string otkey(share->cf_name); otkey.append((const char *)old_dk_chk, old_dk_chk_len); - trx->txn_key_state[otkey] = false; - trx->txn_key_state[ntkey] = true; + trx_set_key_state(trx, otkey, false); + trx_set_key_state(trx, ntkey, true); } } if (share->num_secondary_indexes > 0) { const my_ptrdiff_t nd_ptrdiff = (my_ptrdiff_t)(new_data - table->record[0]); + /* Reuse the per-index dup-check iterators that write_row caches, + invalidating on txn change, so a multi-row UPDATE that shifts a + UNIQUE value does not rebuild the catastrophically expensive + tidesdb_iter_new() (O(num_sstables) merge-heap construction) on + every row. Mirrors the caching in write_row. */ + const uint64_t cur_gen = trx ? trx->txn_generation : 0; for (uint i = 0; i < table->s->keys; i++) { if (share->has_user_pk && i == share->pk_index) continue; @@ -8157,12 +8340,29 @@ int ha_tidesdb::update_row(const uchar *old_data, const uchar *new_data) memcmp(old_prefix, new_prefix, new_prefix_len) == 0) continue; - tidesdb_iter_t *dup_iter = NULL; - int irc = tdb_iter_new_blocking(ha_thd(), txn, share->idx_cfs[i], &dup_iter); - if (irc != TDB_SUCCESS || !dup_iter) + /* Get or create the cached dup-check iterator for this index, + invalidating if the txn changed (commit/reset frees the txn + ops the iterator's MERGE_SOURCE_TXN_OPS depends on). Shared + with write_row via dup_iter_cache_. */ + tidesdb_iter_t *dup_iter = dup_iter_cache_[i]; + if (dup_iter && (dup_iter_txn_[i] != txn || dup_iter_txn_gen_[i] != cur_gen)) { - tmp_restore_column_map(&table->read_set, old_map); - DBUG_RETURN(tdb_rc_to_ha(irc, "update_row dup_iter_new")); + tidesdb_iter_free(dup_iter); + dup_iter = NULL; + dup_iter_cache_[i] = NULL; + } + if (!dup_iter) + { + int irc = tdb_iter_new_blocking(ha_thd(), txn, share->idx_cfs[i], &dup_iter); + if (irc != TDB_SUCCESS || !dup_iter) + { + tmp_restore_column_map(&table->read_set, old_map); + DBUG_RETURN(tdb_rc_to_ha(irc, "update_row dup_iter_new")); + } + dup_iter_cache_[i] = dup_iter; + dup_iter_txn_[i] = txn; + dup_iter_txn_gen_[i] = cur_gen; + dup_iter_count_++; } tidesdb_iter_seek(dup_iter, new_prefix, new_prefix_len); @@ -8180,7 +8380,7 @@ int ha_tidesdb::update_row(const uchar *old_data, const uchar *new_data) memcpy(dup_ref, fk + new_prefix_len, suffix_len); } } - tidesdb_iter_free(dup_iter); + /* Cached iterator is retained for the next row; not freed here. */ if (dup) { @@ -8563,7 +8763,7 @@ int ha_tidesdb::delete_row(const uchar *buf) { std::string tkey(share->cf_name); tkey.append((const char *)dk, dk_len); - trx->txn_key_state[tkey] = false; + trx_set_key_state(trx, tkey, false); } /* We delete secondary index entries in a single consolidated dispatch loop. @@ -8698,7 +8898,9 @@ int ha_tidesdb::delete_all_rows(void) stmt_txn_dirty = false; } - tidesdb_column_family_config_t cfg = build_cf_config(TDB_TABLE_OPTIONS(table)); + const ha_table_option_struct *t_opts = TDB_TABLE_OPTIONS(table); + tidesdb_column_family_config_t cfg = build_cf_config(t_opts); + tidesdb_column_family_config_t data_cfg = data_cf_config(cfg, t_opts && t_opts->encrypted); { std::string cf_name = share->cf_name; @@ -8710,7 +8912,7 @@ int ha_tidesdb::delete_all_rows(void) DBUG_RETURN(tdb_rc_to_ha(rc, "truncate drop_cf")); } - rc = tidesdb_create_column_family(tdb_global, cf_name.c_str(), &cfg); + rc = tidesdb_create_column_family(tdb_global, cf_name.c_str(), &data_cfg); if (rc != TDB_SUCCESS) { sql_print_error("[TIDESDB] truncate: failed to recreate CF '%s' (err=%d)", @@ -8763,9 +8965,10 @@ int ha_tidesdb::delete_all_rows(void) /* Commit the current txn mid-statement and reset it with READ_COMMITTED so the next batch starts fresh. Shared by bulk INSERT/UPDATE/DELETE once - buffered ops cross TIDESDB_BULK_INSERT_BATCH_OPS -- keeps us under - TDB_MAX_TXN_OPS and bounds txn memory. Higher isolation levels would - cause unbounded read-set growth across batches. + buffered ops cross TIDESDB_BULK_INSERT_BATCH_OPS, which bounds the memory the + txn holds before commit since the whole write set lives in the ops array + until then. The reset drops to READ_COMMITTED so the read set does not grow + across batches the way a higher isolation level would. Any cached iterators and dup-check iterators are invalidated, they hold references to MERGE_SOURCE_TXN_OPS that txn_reset clears. @@ -8821,6 +9024,13 @@ int ha_tidesdb::maybe_bulk_commit(tidesdb_trx_t *trx) scan_iter_txn_ = NULL; } free_dup_iter_cache(); + /* The txn was torn down, so the armed statement savepoint and its lock + marker are void; disarm so a later statement rollback falls back to a + full rollback rather than a stale partial one. */ + trx->stmt_savepoint_active = false; + trx->stmt_lock_marker = nullptr; + trx->stmt_key_state_undo.clear(); + trx->stmt_fts_snapshot.clear(); return tdb_rc_to_ha(crc, "bulk_commit"); } @@ -8857,6 +9067,14 @@ int ha_tidesdb::maybe_bulk_commit(tidesdb_trx_t *trx) } free_dup_iter_cache(); scan_txn = trx->txn; + /* The mid-statement commit reset the txn and released every row lock, so the + armed statement savepoint and its lock marker no longer mean anything. + Disarm it -- the remainder of this statement can no longer be rolled back + atomically (the committed rows are durable), matching prior behavior. */ + trx->stmt_savepoint_active = false; + trx->stmt_lock_marker = nullptr; + trx->stmt_key_state_undo.clear(); + trx->stmt_fts_snapshot.clear(); return 0; } @@ -9166,7 +9384,20 @@ int ha_tidesdb::info(uint flag) tidesdb_stats_t *st = NULL; if (tidesdb_get_stats(share->cf, &st) == TDB_SUCCESS && st) { - share->cached_records.store(st->total_keys, std::memory_order_relaxed); + /* Prefer the distinct-key cardinality estimate for the row + count. Within an SSTable it counts each key once no matter + how many MVCC versions it holds, which total_keys counts + separately, so it is a tighter row estimate for the optimizer + (it still upper-bounds a key spread across several SSTables). + Fall back to total_keys when the library cannot supply it. */ + uint64_t rows = st->total_keys; + uint64_t est = 0; + /* The aggregate is always a real sum -- the per-SSTable + TDB_DISTINCT_KEYS_UNKNOWN sentinel is resolved to that + SSTable's entry count inside the library -- so success alone + means the value is usable. */ + if (tidesdb_cf_estimate_cardinality(share->cf, &est) == TDB_SUCCESS) rows = est; + share->cached_records.store(rows, std::memory_order_relaxed); /* total_data_size only counts SSTable klog+vlog; memtable_size holds the active memtable footprint. Sum both so that @@ -9885,7 +10116,18 @@ ulong ha_tidesdb::index_flags(uint idx, uint part, bool all_parts) const if (table_share && table_share->primary_key != MAX_KEY && idx == table_share->primary_key) flags |= HA_CLUSTERED_INDEX; else - flags |= HA_KEYREAD_ONLY; + { + /* The server builds field->part_of_key (the covering-index bitmap) by + calling index_flags(idx, part, 0) once per key part, so advertise + HA_KEYREAD_ONLY for a part only when that part's field reconstructs + from its sort key via decode_sort_key_part. A query that reads only + decodable columns of a mixed-type index still gets an index-only + plan; a read of an undecodable column (VARCHAR, DECIMAL, floating + point, multi-byte CHAR) leaves part_of_key clear so the optimizer + prices the primary-key row fetch instead of a phantom covering scan + that try_keyread_from_index would fall back on at runtime. */ + if (index_part_is_decodable(table_share, idx, part)) flags |= HA_KEYREAD_ONLY; + } return flags; } @@ -10525,6 +10767,16 @@ int ha_tidesdb::external_lock(THD *thd, int lock_type) trans_register_ha(thd, false, ht, 0); if (!is_autocommit) trans_register_ha(thd, true, ht, 0); + + /* Arm a statement savepoint so a statement error inside BEGIN...COMMIT + rolls back only this statement, not the whole transaction. Only for + real multi-statement transactions -- autocommit and DDL statements + are their own transaction, where statement rollback is already a full + rollback. external_lock fires once per table per statement, so the + !stmt_savepoint_active guard arms it exactly once, at the first table, + before any of the statement's row writes. */ + if (!is_autocommit && !is_ddl && trx->txn && !trx->stmt_savepoint_active) + stmt_savepoint_arm(trx); } else { @@ -11169,6 +11421,7 @@ bool ha_tidesdb::commit_inplace_alter_table(TABLE *altered_table, Alter_inplace_ } } } + share->num_secondary_indexes = 0; for (uint i = 0; i < share->idx_cfs.size(); i++) if (share->idx_cfs[i]) share->num_secondary_indexes++; @@ -11178,12 +11431,14 @@ bool ha_tidesdb::commit_inplace_alter_table(TABLE *altered_table, Alter_inplace_ of only being persisted in the .frm. */ if (ha_alter_info->handler_flags & ALTER_CHANGE_CREATE_OPTION) { - tidesdb_column_family_config_t cfg = build_cf_config(TDB_TABLE_OPTIONS(altered_table)); + const ha_table_option_struct *a_opts = TDB_TABLE_OPTIONS(altered_table); + tidesdb_column_family_config_t cfg = build_cf_config(a_opts); + tidesdb_column_family_config_t data_cfg = data_cf_config(cfg, a_opts && a_opts->encrypted); /* Main data CF */ if (share->cf) { - int rc = tidesdb_cf_update_runtime_config(share->cf, &cfg, 1); + int rc = tidesdb_cf_update_runtime_config(share->cf, &data_cfg, 1); if (rc != TDB_SUCCESS) sql_print_warning( "[TIDESDB] ALTER: failed to update runtime config for " @@ -11557,8 +11812,8 @@ static long long srv_stat_compaction_count; tidesdb_show_status can read them directly. Their definitions live up there. */ -#define TIDESQL_VERSION_STR "4.5.8" -#define TIDESQL_VERSION_HEX 0x40508 +#define TIDESQL_VERSION_STR "4.5.9" +#define TIDESQL_VERSION_HEX 0x40509 static const char *srv_stat_version = TIDESQL_VERSION_STR; static long long srv_stat_version_hex = TIDESQL_VERSION_HEX; diff --git a/tidesdb/ha_tidesdb.h b/tidesdb/ha_tidesdb.h index 8b58cc0f..4caf7471 100644 --- a/tidesdb/ha_tidesdb.h +++ b/tidesdb/ha_tidesdb.h @@ -564,17 +564,31 @@ struct tidesdb_trx_t std::vector fts_meta_pending; bool fts_meta_dirty{false}; - /* Uniqueness probing must not seed the WRITE txn's read-set, a point-get - of an absent key records read-seq 0, which the library's first-committer - reservation then uses as the conflict base, so a stale slot from any - prior write of that key trips a spurious conflict under load. Instead we - probe committed data on a dedicated READ_COMMITTED txn (no read-set, never - pins the snapshot floor) and track same-txn pending existence here: + /* Same-txn pending key existence, consulted by probe_pk_exists before its + non-tracking tidesdb_txn_contains check on the write txn. Committed data + alone cannot show what this txn has staged, so we record it here: true = inserted this txn (re-insert is a duplicate), false = deleted this txn (re-insert is allowed though committed has it). Keyed by cf_name + primary-key bytes; cleared at commit/rollback. */ - tidesdb_txn_t *dup_rtxn{nullptr}; std::unordered_map txn_key_state; + + /* Statement-atomicity bookkeeping. A "stmt" library savepoint is armed at + statement start (external_lock) inside a multi-statement transaction so a + statement error rolls back only its own effects, not the whole txn. The + library savepoint reverts the txn op array; these members revert the + plugin-side shadow state to the same statement boundary. + + stmt_lock_marker is held_locks_head at statement start; locks acquired + during the statement sit above it and are released to it on rollback. + stmt_key_state_undo journals prior txn_key_state values per mutation + (prior: -1 absent, 0 false, 1 true) so we revert in O(statement writes) + rather than snapshotting the whole (potentially huge) map. + stmt_fts_snapshot is a copy of the small fts_meta_pending vector taken at + statement start and restored wholesale on rollback. */ + tdb_lock_request_t *stmt_lock_marker{nullptr}; + std::vector> stmt_key_state_undo; + std::vector stmt_fts_snapshot; + bool stmt_fts_dirty_snapshot{false}; }; /* @@ -964,7 +978,7 @@ class ha_tidesdb : public handler /* Bulk UPDATE / DELETE hints -- let multi-row UPDATE/DELETE share the same mid-txn commit batching as bulk INSERT so long statements don't - blow past TDB_MAX_TXN_OPS or balloon txn memory. */ + balloon the memory the txn buffers before commit. */ bool start_bulk_update() override; int end_bulk_update() override; int bulk_update_row(const uchar *old_data, const uchar *new_data, From 23dae79de76929b1d9230b3f5df82dd7991a343a Mon Sep 17 00:00:00 2001 From: Alex Gaetano Padula Date: Tue, 7 Jul 2026 18:14:25 -0400 Subject: [PATCH 2/2] stop pinning xcode 16.4 in the macos ci steps --- .github/workflows/mariadb-test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/mariadb-test.yml b/.github/workflows/mariadb-test.yml index 9fc0c725..366a04eb 100644 --- a/.github/workflows/mariadb-test.yml +++ b/.github/workflows/mariadb-test.yml @@ -205,7 +205,6 @@ jobs: - name: Install TidesDB library (macOS) if: runner.os == 'macOS' run: | - sudo xcode-select -s /Applications/Xcode_16.4.app/Contents/Developer export SDKROOT=$(xcrun --show-sdk-path) rm -rf tidesdb-lib @@ -296,7 +295,6 @@ jobs: if: runner.os == 'macOS' working-directory: mariadb-server run: | - sudo xcode-select -s /Applications/Xcode_16.4.app/Contents/Developer sudo rm -rf /Library/Developer/CommandLineTools/SDKs/MacOSX*.sdk