diff --git a/.github/workflows/mariadb-test.yml b/.github/workflows/mariadb-test.yml index 9fc0c725..366a04eb 100644 --- a/.github/workflows/mariadb-test.yml +++ b/.github/workflows/mariadb-test.yml @@ -205,7 +205,6 @@ jobs: - name: Install TidesDB library (macOS) if: runner.os == 'macOS' run: | - sudo xcode-select -s /Applications/Xcode_16.4.app/Contents/Developer export SDKROOT=$(xcrun --show-sdk-path) rm -rf tidesdb-lib @@ -296,7 +295,6 @@ jobs: if: runner.os == 'macOS' working-directory: mariadb-server run: | - sudo xcode-select -s /Applications/Xcode_16.4.app/Contents/Developer sudo rm -rf /Library/Developer/CommandLineTools/SDKs/MacOSX*.sdk diff --git a/mysql-test/suite/tidesdb/r/tidesdb_encrypted_no_compress.result b/mysql-test/suite/tidesdb/r/tidesdb_encrypted_no_compress.result new file mode 100644 index 00000000..20410fc9 --- /dev/null +++ b/mysql-test/suite/tidesdb/r/tidesdb_encrypted_no_compress.result @@ -0,0 +1,64 @@ +# +# ============================================ +# Encrypted table: data CF compression forced off, index CF keeps it +# ============================================ +# +CREATE TABLE enc_lz4 ( +id INT PRIMARY KEY, +u INT, +v VARCHAR(200), +KEY k_u (u) +) ENGINE=TIDESDB `ENCRYPTED`=YES COMPRESSION='LZ4'; +# the table still reports the requested COMPRESSION option to the user +SHOW CREATE TABLE enc_lz4; +Table Create Table +enc_lz4 CREATE TABLE `enc_lz4` ( + `id` int(11) NOT NULL, + `u` int(11) DEFAULT NULL, + `v` varchar(200) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `k_u` (`u`) +) ENGINE=TidesDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci `ENCRYPTED`=YES `COMPRESSION`='LZ4' +# data CF: compression_algorithm = NONE (ciphertext is incompressible) +compression_algorithm = NONE +# index CF: compression_algorithm = LZ4 (unencrypted keys keep compression) +compression_algorithm = LZ4 +# +# ============================================ +# Control: non-encrypted table keeps the selected compression on its data CF +# ============================================ +# +CREATE TABLE plain_lz4 ( +id INT PRIMARY KEY, +v VARCHAR(200) +) ENGINE=TIDESDB COMPRESSION='LZ4'; +# data CF: compression_algorithm = LZ4 (not encrypted, so compression stays) +compression_algorithm = LZ4 +# +# ============================================ +# Data still round-trips under an encrypted, compression-disabled CF +# ============================================ +# +INSERT INTO enc_lz4 VALUES (1, 10, 'alpha'), (2, 20, 'beta'), (3, 30, 'gamma'); +UPDATE enc_lz4 SET v = 'delta' WHERE id = 2; +DELETE FROM enc_lz4 WHERE id = 3; +SELECT * FROM enc_lz4 ORDER BY id; +id u v +1 10 alpha +2 20 delta +# covering index scan on the still-compressed index CF +SELECT id FROM enc_lz4 WHERE u = 10; +id +1 +# +# ============================================ +# TRUNCATE recreates the data CF with compression still off +# ============================================ +# +TRUNCATE TABLE enc_lz4; +compression_algorithm = NONE +compression_algorithm = LZ4 +DROP TABLE enc_lz4; +DROP TABLE plain_lz4; +# +# Done. diff --git a/mysql-test/suite/tidesdb/r/tidesdb_index_stats.result b/mysql-test/suite/tidesdb/r/tidesdb_index_stats.result index 11aeb88a..8c4a0906 100644 --- a/mysql-test/suite/tidesdb/r/tidesdb_index_stats.result +++ b/mysql-test/suite/tidesdb/r/tidesdb_index_stats.result @@ -77,7 +77,7 @@ test.t_stats analyze status OK # After ANALYZE, the optimizer should estimate ~100 rows for k=0 EXPLAIN SELECT * FROM t_stats WHERE k = 0; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t_stats ref k_idx k_idx 4 const 2 +1 SIMPLE t_stats ALL k_idx NULL NULL NULL 200 Using where DROP TABLE t_stats; # # ============================================ diff --git a/mysql-test/suite/tidesdb/r/tidesdb_info_schema.result b/mysql-test/suite/tidesdb/r/tidesdb_info_schema.result index 716c0583..874a11fc 100644 --- a/mysql-test/suite/tidesdb/r/tidesdb_info_schema.result +++ b/mysql-test/suite/tidesdb/r/tidesdb_info_schema.result @@ -7,20 +7,20 @@ INSERT INTO t_info_schema VALUES (1, REPEAT('a', 100)); INSERT INTO t_info_schema VALUES (2, REPEAT('b', 100)); INSERT INTO t_info_schema VALUES (3, REPEAT('c', 100)); # =+=+= data_length must be non-zero =+=+= -FAIL: DATA_LENGTH is 0 +OK: DATA_LENGTH > 0 # =+=+= table_rows must reflect inserted rows =+=+= -FAIL: TABLE_ROWS < 3 +OK: TABLE_ROWS >= 3 # =+=+= add secondary index and check index_length =+=+= ALTER TABLE t_info_schema ADD INDEX idx_val (val); SELECT COUNT(*) FROM t_info_schema; COUNT(*) 3 -FAIL: INDEX_LENGTH is 0 +OK: INDEX_LENGTH > 0 # =+=+= verify after bulk insert =+=+= SELECT COUNT(*) FROM t_info_schema; COUNT(*) 200 -FAIL: DATA_LENGTH is 0 after bulk insert +OK: DATA_LENGTH > 0 after bulk insert # =+=+= create_time must be non-null =+=+= OK: CREATE_TIME is set # =+=+= update_time must be non-null after DML =+=+= diff --git a/mysql-test/suite/tidesdb/r/tidesdb_keyread_decodable.result b/mysql-test/suite/tidesdb/r/tidesdb_keyread_decodable.result new file mode 100644 index 00000000..8f0b09af --- /dev/null +++ b/mysql-test/suite/tidesdb/r/tidesdb_keyread_decodable.result @@ -0,0 +1,111 @@ +# +# ============================================ +# Decodable column types: covering (Using index) IS offered +# ============================================ +# +CREATE TABLE cov_ok ( +id INT PRIMARY KEY, +i INT, +b BIGINT, +d DATE, +ts DATETIME, +c CHAR(8) CHARACTER SET binary, +KEY k_i (i), +KEY k_b (b), +KEY k_d (d), +KEY k_ts (ts), +KEY k_c (c) +) ENGINE=TIDESDB; +INSERT INTO cov_ok VALUES +(1, 10, 100, '2020-01-01', '2020-01-01 10:00:00', 'aaa'), +(2, 20, 200, '2021-02-02', '2021-02-02 11:00:00', 'bbb'), +(3, 30, 300, '2022-03-03', '2022-03-03 12:00:00', 'ccc'); +EXPLAIN SELECT i FROM cov_ok FORCE INDEX (k_i) WHERE i = 20; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_ok ref k_i k_i 5 const # Using index +EXPLAIN SELECT b FROM cov_ok FORCE INDEX (k_b) WHERE b = 200; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_ok ref k_b k_b 9 const # Using index +EXPLAIN SELECT d FROM cov_ok FORCE INDEX (k_d) WHERE d = '2021-02-02'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_ok ref k_d k_d 4 const # Using index +EXPLAIN SELECT ts FROM cov_ok FORCE INDEX (k_ts) WHERE ts = '2021-02-02 11:00:00'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_ok ref k_ts k_ts 6 const # Using index +EXPLAIN SELECT c FROM cov_ok FORCE INDEX (k_c) WHERE c = 'bbb'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_ok ref k_c k_c 9 const # Using where; Using index +# +# ============================================ +# Undecodable column types: covering is NOT offered (no Using index) +# ============================================ +# +CREATE TABLE cov_no ( +id INT PRIMARY KEY, +s VARCHAR(50), +u CHAR(8) CHARACTER SET utf8mb4, +m DECIMAL(10,2), +f DOUBLE, +KEY k_s (s), +KEY k_u (u), +KEY k_m (m), +KEY k_f (f) +) ENGINE=TIDESDB; +INSERT INTO cov_no VALUES +(1, 'alpha', 'x1', 1.50, 1.5), +(2, 'beta', 'x2', 2.50, 2.5), +(3, 'gamma', 'x3', 3.50, 3.5); +EXPLAIN SELECT s FROM cov_no FORCE INDEX (k_s) WHERE s = 'beta'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_no ref k_s k_s 203 const # Using where +EXPLAIN SELECT u FROM cov_no FORCE INDEX (k_u) WHERE u = 'x2'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_no ref k_u k_u 33 const # Using where +EXPLAIN SELECT m FROM cov_no FORCE INDEX (k_m) WHERE m = 2.50; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_no ref k_m k_m 6 const # +EXPLAIN SELECT f FROM cov_no FORCE INDEX (k_f) WHERE f = 2.5; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_no ref k_f k_f 9 const # +# results are still correct on the non-covering (PK fetch) path +SELECT s FROM cov_no FORCE INDEX (k_s) WHERE s = 'beta'; +s +beta +SELECT m FROM cov_no FORCE INDEX (k_m) WHERE m = 2.50; +m +2.50 +# +# ============================================ +# Composite index: covering decided per key part, not per whole index +# ============================================ +# +CREATE TABLE cov_mix ( +id INT PRIMARY KEY, +a INT, +b INT, +s VARCHAR(50), +KEY k_ab (a, b), +KEY k_as (a, s) +) ENGINE=TIDESDB; +INSERT INTO cov_mix VALUES (1, 1, 10, 'x'), (2, 2, 20, 'y'), (3, 3, 30, 'z'); +# (a INT, b INT): both decodable -> Using index +EXPLAIN SELECT a, b FROM cov_mix FORCE INDEX (k_ab) WHERE a = 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_mix ref k_ab k_ab 5 const # Using index +# (a INT, s VARCHAR), reading only the decodable a -> still Using index +EXPLAIN SELECT a FROM cov_mix FORCE INDEX (k_as) WHERE a = 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_mix ref k_as k_as 5 const # Using index +# (a INT, s VARCHAR), reading the undecodable s -> no Using index +EXPLAIN SELECT a, s FROM cov_mix FORCE INDEX (k_as) WHERE a = 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE cov_mix ref k_as k_as 5 const # +# result on the s (non-covering) path is still correct +SELECT a, s FROM cov_mix FORCE INDEX (k_as) WHERE a = 2; +a s +2 y +DROP TABLE cov_ok; +DROP TABLE cov_no; +DROP TABLE cov_mix; +# +# Done. diff --git a/mysql-test/suite/tidesdb/r/tidesdb_mrr.result b/mysql-test/suite/tidesdb/r/tidesdb_mrr.result index bea30b41..763cd938 100644 --- a/mysql-test/suite/tidesdb/r/tidesdb_mrr.result +++ b/mysql-test/suite/tidesdb/r/tidesdb_mrr.result @@ -9,7 +9,7 @@ INSERT INTO t_pk VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e'), # Confirm the optimizer actually picks Rowid-ordered scan (MRR). EXPLAIN SELECT * FROM t_pk WHERE id IN (7, 2, 9, 3, 5); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t_pk range PRIMARY # 4 NULL 2 Using where +1 SIMPLE t_pk range PRIMARY # 4 NULL 5 Using where # Unsorted IN-list; MRR must still return the right rows. SELECT * FROM t_pk WHERE id IN (7, 2, 9, 3, 5) ORDER BY id; id v diff --git a/mysql-test/suite/tidesdb/r/tidesdb_online_ddl.result b/mysql-test/suite/tidesdb/r/tidesdb_online_ddl.result index 2b8d42bb..b5c9b3d3 100644 --- a/mysql-test/suite/tidesdb/r/tidesdb_online_ddl.result +++ b/mysql-test/suite/tidesdb/r/tidesdb_online_ddl.result @@ -36,8 +36,8 @@ t_ddl CREATE TABLE `t_ddl` ( ALTER TABLE t_ddl ADD INDEX idx_a (a), ALGORITHM=INPLACE; SHOW INDEX FROM t_ddl; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored -t_ddl 0 PRIMARY 1 id A 2 NULL NULL LSM NO -t_ddl 1 idx_a 1 a A 2 NULL NULL YES LSM NO +t_ddl 0 PRIMARY 1 id A 6 NULL NULL LSM NO +t_ddl 1 idx_a 1 a A 6 NULL NULL YES LSM NO # Verify index is usable SELECT id, a FROM t_ddl WHERE a = 10 ORDER BY id; id a @@ -52,9 +52,9 @@ id a ALTER TABLE t_ddl ADD INDEX idx_c (c), ALGORITHM=INPLACE; SHOW INDEX FROM t_ddl; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored -t_ddl 0 PRIMARY 1 id A 2 NULL NULL LSM NO -t_ddl 1 idx_a 1 a A 2 NULL NULL YES LSM NO -t_ddl 1 idx_c 1 c A 2 NULL NULL YES LSM NO +t_ddl 0 PRIMARY 1 id A 6 NULL NULL LSM NO +t_ddl 1 idx_a 1 a A 6 NULL NULL YES LSM NO +t_ddl 1 idx_c 1 c A 6 NULL NULL YES LSM NO EXPLAIN SELECT id, c FROM t_ddl WHERE c = 200; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t_ddl ref idx_c idx_c 5 const 1 Using index @@ -65,8 +65,8 @@ id c ALTER TABLE t_ddl DROP INDEX idx_a, ALGORITHM=INPLACE; SHOW INDEX FROM t_ddl; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored -t_ddl 0 PRIMARY 1 id A 2 NULL NULL LSM NO -t_ddl 1 idx_c 1 c A 2 NULL NULL YES LSM NO +t_ddl 0 PRIMARY 1 id A 6 NULL NULL LSM NO +t_ddl 1 idx_c 1 c A 6 NULL NULL YES LSM NO # Verify remaining index still works SELECT id, c FROM t_ddl WHERE c = 300; id c @@ -75,8 +75,8 @@ id c ALTER TABLE t_ddl ADD INDEX idx_a2 (a), DROP INDEX idx_c, ALGORITHM=INPLACE; SHOW INDEX FROM t_ddl; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored -t_ddl 0 PRIMARY 1 id A 2 NULL NULL LSM NO -t_ddl 1 idx_a2 1 a A 2 NULL NULL YES LSM NO +t_ddl 0 PRIMARY 1 id A 6 NULL NULL LSM NO +t_ddl 1 idx_a2 1 a A 6 NULL NULL YES LSM NO EXPLAIN SELECT id, a FROM t_ddl WHERE a = 20; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t_ddl ref idx_a2 idx_a2 5 const 1 Using index diff --git a/mysql-test/suite/tidesdb/r/tidesdb_per_index_btree.result b/mysql-test/suite/tidesdb/r/tidesdb_per_index_btree.result index 3400c01d..0f80f9bd 100644 --- a/mysql-test/suite/tidesdb/r/tidesdb_per_index_btree.result +++ b/mysql-test/suite/tidesdb/r/tidesdb_per_index_btree.result @@ -12,9 +12,9 @@ INSERT INTO t1 VALUES (1,10,100),(2,20,200),(3,30,300); # idx_a should show BTREE, idx_b should show LSM SHOW KEYS FROM t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored -t1 0 PRIMARY 1 id A 2 NULL NULL LSM NO -t1 1 idx_a 1 a A 2 NULL NULL YES BTREE NO -t1 1 idx_b 1 b A 2 NULL NULL YES LSM NO +t1 0 PRIMARY 1 id A 3 NULL NULL LSM NO +t1 1 idx_a 1 a A 3 NULL NULL YES BTREE NO +t1 1 idx_b 1 b A 3 NULL NULL YES LSM NO SELECT * FROM t1 WHERE a = 20; id a b 2 20 200 diff --git a/mysql-test/suite/tidesdb/r/tidesdb_row_cardinality.result b/mysql-test/suite/tidesdb/r/tidesdb_row_cardinality.result new file mode 100644 index 00000000..017f072b --- /dev/null +++ b/mysql-test/suite/tidesdb/r/tidesdb_row_cardinality.result @@ -0,0 +1,51 @@ +CREATE TABLE t_a (id INT PRIMARY KEY, v INT, KEY k_v (v)) ENGINE=TIDESDB; +CREATE TABLE t_b (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB; +ANALYZE TABLE t_a; +Table Op Msg_type Msg_text +test.t_a analyze status Engine-independent statistics collected +test.t_a analyze Note [TIDESDB] CF 'test__t_a' total_keys=200 data_size=0 bytes memtable=4600 bytes levels=1 read_amp=1.00 cache_hit=0.0% +test.t_a analyze Note [TIDESDB] avg_key=6.9 bytes avg_value=16.1 bytes +test.t_a analyze Note [TIDESDB] level 1 sstables=0 size=0 bytes keys=0 +test.t_a analyze Note [TIDESDB] WA user=3800 wal=0 flush=0 (0 ssts) compact_write=0 (0 ssts) compact_read=0 ratio=0.00x +test.t_a analyze Note [TIDESDB] idx CF 'test__t_a__idx_k_v' keys=200 data_size=0 bytes levels=1 +test.t_a analyze Note [TIDESDB] idx 'k_v' sampled=200 distinct=10 rec_per_key=20 +test.t_a analyze status OK +ANALYZE TABLE t_b; +Table Op Msg_type Msg_text +test.t_b analyze status Engine-independent statistics collected +test.t_b analyze Note [TIDESDB] CF 'test__t_b' total_keys=75 data_size=0 bytes memtable=1725 bytes levels=1 read_amp=1.00 cache_hit=0.0% +test.t_b analyze Note [TIDESDB] avg_key=6.9 bytes avg_value=16.1 bytes +test.t_b analyze Note [TIDESDB] level 1 sstables=0 size=0 bytes keys=0 +test.t_b analyze Note [TIDESDB] WA user=1425 wal=0 flush=0 (0 ssts) compact_write=0 (0 ssts) compact_read=0 ratio=0.00x +test.t_b analyze status OK +# each table reports its own distinct row count, not the shared-memtable sum +# t_a -> 200 +SELECT TABLE_ROWS FROM information_schema.TABLES +WHERE table_schema = DATABASE() AND table_name = 't_a'; +TABLE_ROWS +200 +# t_b -> 75 +SELECT TABLE_ROWS FROM information_schema.TABLES +WHERE table_schema = DATABASE() AND table_name = 't_b'; +TABLE_ROWS +75 +# inserting more rows raises the estimate on the next refresh +ANALYZE TABLE t_a; +Table Op Msg_type Msg_text +test.t_a analyze status Engine-independent statistics collected +test.t_a analyze Note [TIDESDB] CF 'test__t_a' total_keys=300 data_size=0 bytes memtable=6900 bytes levels=1 read_amp=1.00 cache_hit=0.0% +test.t_a analyze Note [TIDESDB] avg_key=6.9 bytes avg_value=16.1 bytes +test.t_a analyze Note [TIDESDB] level 1 sstables=0 size=0 bytes keys=0 +test.t_a analyze Note [TIDESDB] WA user=5700 wal=0 flush=0 (0 ssts) compact_write=0 (0 ssts) compact_read=0 ratio=0.00x +test.t_a analyze Note [TIDESDB] idx CF 'test__t_a__idx_k_v' keys=300 data_size=0 bytes levels=1 +test.t_a analyze Note [TIDESDB] idx 'k_v' sampled=300 distinct=10 rec_per_key=30 +test.t_a analyze status OK +# t_a -> 300 +SELECT TABLE_ROWS FROM information_schema.TABLES +WHERE table_schema = DATABASE() AND table_name = 't_a'; +TABLE_ROWS +300 +DROP TABLE t_a; +DROP TABLE t_b; +# +# Done. diff --git a/mysql-test/suite/tidesdb/r/tidesdb_secondary_gt_range.result b/mysql-test/suite/tidesdb/r/tidesdb_secondary_gt_range.result new file mode 100644 index 00000000..d2589d10 --- /dev/null +++ b/mysql-test/suite/tidesdb/r/tidesdb_secondary_gt_range.result @@ -0,0 +1,78 @@ +# +# ============================================ +# > X excludes all X duplicates, includes only greater values +# ============================================ +# +CREATE TABLE t1 (id INT PRIMARY KEY, u INT, KEY k_u (u)) ENGINE=TIDESDB; +INSERT INTO t1 VALUES +(1, 2), (2, 2), (3, 2), (4, 2), (5, 2), +(6, 3), (7, 3), +(8, 4), +(9, 1), (10, 1); +# u > 2 -> only the u=3 and u=4 rows +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 2 ORDER BY u, id; +id u +6 3 +7 3 +8 4 +# u >= 2 (control) -> includes the u=2 rows +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u >= 2 ORDER BY u, id; +id u +1 2 +2 2 +3 2 +4 2 +5 2 +6 3 +7 3 +8 4 +# u > 4 (greatest value) -> empty +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 4 ORDER BY u, id; +id u +# bounded range u > 2 AND u < 4 -> only u=3 +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 2 AND u < 4 ORDER BY id; +id u +6 3 +7 3 +# +# ============================================ +# Pathological: a row whose PK encodes to the max key bytes (INT max) +# sits at X's upper bound and must still be excluded by > X +# ============================================ +# +INSERT INTO t1 VALUES (2147483647, 2); +# the max-PK row has u=2, so u > 2 must NOT return it +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 2 ORDER BY u, id; +id u +6 3 +7 3 +8 4 +# u >= 2 must include it +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u >= 2 AND u < 3 ORDER BY id; +id u +1 2 +2 2 +3 2 +4 2 +5 2 +2147483647 2 +# +# ============================================ +# Composite index: > on the leading part +# ============================================ +# +CREATE TABLE t2 (id INT PRIMARY KEY, a INT, b INT, KEY k_ab (a, b)) ENGINE=TIDESDB; +INSERT INTO t2 VALUES +(1, 1, 10), (2, 1, 20), (3, 1, 30), +(4, 2, 10), (5, 2, 20), +(6, 3, 99); +# a > 1 -> only a=2 and a=3 rows +SELECT id, a, b FROM t2 FORCE INDEX (k_ab) WHERE a > 1 ORDER BY a, b; +id a b +4 2 10 +5 2 20 +6 3 99 +DROP TABLE t1; +DROP TABLE t2; +# +# Done. diff --git a/mysql-test/suite/tidesdb/r/tidesdb_stmt_atomicity.result b/mysql-test/suite/tidesdb/r/tidesdb_stmt_atomicity.result new file mode 100644 index 00000000..129a69fb --- /dev/null +++ b/mysql-test/suite/tidesdb/r/tidesdb_stmt_atomicity.result @@ -0,0 +1,162 @@ +# +# ============================================ +# TEST 1: failed statement leaves earlier ones intact (PRIMARY KEY) +# ============================================ +# +CREATE TABLE t1 (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB; +START TRANSACTION; +INSERT INTO t1 VALUES (1, 10); +INSERT INTO t1 VALUES (2, 20); +# row 3 inserts, then row 1 duplicates -> whole statement rolls back +INSERT INTO t1 VALUES (3, 30), (1, 99); +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +# inside the txn: 1 and 2 survive, 3 is gone, 1 is unchanged (v=10) +SELECT * FROM t1 ORDER BY id; +id v +1 10 +2 20 +COMMIT; +# after commit: same result persisted +SELECT * FROM t1 ORDER BY id; +id v +1 10 +2 20 +# +# ============================================ +# TEST 2: re-insert of a rolled-back key succeeds (txn_key_state revert) +# ============================================ +# +START TRANSACTION; +INSERT INTO t1 VALUES (5, 50); +# row 6 inserts, then row 5 duplicates -> statement rolls back, 6 undone +INSERT INTO t1 VALUES (6, 60), (5, 999); +ERROR 23000: Duplicate entry '5' for key 'PRIMARY' +# 6 was rolled back, so re-inserting it must NOT report a false duplicate +INSERT INTO t1 VALUES (6, 66); +COMMIT; +SELECT * FROM t1 ORDER BY id; +id v +1 10 +2 20 +5 50 +6 66 +# +# ============================================ +# TEST 3: UNIQUE secondary index revert + re-insert +# ============================================ +# +CREATE TABLE t2 (id INT PRIMARY KEY, u INT UNIQUE, v INT) ENGINE=TIDESDB; +START TRANSACTION; +INSERT INTO t2 VALUES (1, 100, 1); +# row (2,200) inserts, then u=100 duplicates -> statement rolls back +INSERT INTO t2 VALUES (2, 200, 2), (3, 100, 3); +ERROR 23000: Duplicate entry '100' for key 'u' +# the rolled-back unique value 200 must be free to insert again +INSERT INTO t2 VALUES (2, 200, 2); +COMMIT; +SELECT * FROM t2 ORDER BY id; +id u v +1 100 1 +2 200 2 +# +# ============================================ +# TEST 4: failed UPDATE leaves earlier UPDATE intact +# ============================================ +# +START TRANSACTION; +UPDATE t1 SET v = v + 1; +# moving id 2 onto existing id 1 duplicates the PK -> statement rolls back +UPDATE t1 SET id = 1 WHERE id = 2; +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +# first UPDATE's increment survives; row 2 keeps its id +SELECT * FROM t1 ORDER BY id; +id v +1 11 +2 21 +5 51 +6 67 +COMMIT; +SELECT * FROM t1 ORDER BY id; +id v +1 11 +2 21 +5 51 +6 67 +# +# ============================================ +# TEST 5: statement rollback preserves the transaction snapshot +# ============================================ +# +CREATE TABLE t3 (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB; +INSERT INTO t3 VALUES (1, 1); +connect con1, localhost, root,,; +connection con1; +BEGIN; +# pin con1's snapshot +SELECT * FROM t3 ORDER BY id; +id v +1 1 +connection default; +INSERT INTO t3 VALUES (2, 2); +connection con1; +# a failed statement inside con1's txn must roll back only itself... +INSERT INTO t3 VALUES (3, 3), (1, 3); +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +# ...and must NOT shift the snapshot: con1 still must not see row 2 +SELECT * FROM t3 ORDER BY id; +id v +1 1 +COMMIT; +# after commit the snapshot is released, row 2 becomes visible +SELECT * FROM t3 ORDER BY id; +id v +1 1 +2 2 +disconnect con1; +connection default; +# +# ============================================ +# TEST 6: FULLTEXT index stays consistent after a statement rollback +# ============================================ +# +CREATE TABLE ft (id INT PRIMARY KEY, body TEXT, FULLTEXT(body)) ENGINE=TIDESDB; +START TRANSACTION; +INSERT INTO ft VALUES (1, 'alpha beta'); +# row 2 (gamma delta) inserts, then id 1 duplicates -> statement rolls back +INSERT INTO ft VALUES (2, 'gamma delta'), (1, 'dup'); +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +COMMIT; +# gamma was rolled back -> no match; alpha (committed row 1) still matches +SELECT id FROM ft WHERE MATCH(body) AGAINST('gamma'); +id +SELECT id FROM ft WHERE MATCH(body) AGAINST('alpha'); +id +1 +# a fresh insert of the rolled-back term indexes cleanly +INSERT INTO ft VALUES (2, 'gamma delta'); +SELECT id FROM ft WHERE MATCH(body) AGAINST('gamma'); +id +2 +# +# ============================================ +# TEST 7: user SAVEPOINT still works alongside statement savepoints +# ============================================ +# +START TRANSACTION; +INSERT INTO t1 VALUES (40, 400); +SAVEPOINT a; +INSERT INTO t1 VALUES (41, 410); +ROLLBACK TO SAVEPOINT a; +# 40 survives, 41 undone by the user rollback +INSERT INTO t1 VALUES (42, 420); +COMMIT; +SELECT id FROM t1 WHERE id >= 40 ORDER BY id; +id +40 +42 +DROP TABLE ft; +DROP TABLE t3; +DROP TABLE t2; +DROP TABLE t1; +# +# Done. diff --git a/mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.opt b/mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.opt new file mode 100644 index 00000000..5737dfca --- /dev/null +++ b/mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.opt @@ -0,0 +1,2 @@ +--plugin-load-add=file_key_management +--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt diff --git a/mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.test b/mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.test new file mode 100644 index 00000000..d726a6c6 --- /dev/null +++ b/mysql-test/suite/tidesdb/t/tidesdb_encrypted_no_compress.test @@ -0,0 +1,74 @@ +--source include/have_tidesdb.inc +--source include/not_embedded.inc + +# +# An encrypted table stores ciphertext in its data column family. Ciphertext +# does not compress, so the engine forces the data CF's compression off even +# when the table selects a compression algorithm, avoiding wasted CPU on every +# flush and compaction. Secondary-index CFs hold unencrypted comparable keys +# and must keep the selected compression. +# +# We verify by reading the per-CF config.ini the library writes at create time. +# The data home is the sibling "tidesdb_data" directory of the server datadir. +# + +--echo # +--echo # ============================================ +--echo # Encrypted table: data CF compression forced off, index CF keeps it +--echo # ============================================ +--echo # +CREATE TABLE enc_lz4 ( + id INT PRIMARY KEY, + u INT, + v VARCHAR(200), + KEY k_u (u) +) ENGINE=TIDESDB `ENCRYPTED`=YES COMPRESSION='LZ4'; + +--echo # the table still reports the requested COMPRESSION option to the user +SHOW CREATE TABLE enc_lz4; + +--echo # data CF: compression_algorithm = NONE (ciphertext is incompressible) +--exec grep '^compression_algorithm' $MYSQLTEST_VARDIR/mysqld.1/tidesdb_data/test__enc_lz4/config.ini +--echo # index CF: compression_algorithm = LZ4 (unencrypted keys keep compression) +--exec grep '^compression_algorithm' $MYSQLTEST_VARDIR/mysqld.1/tidesdb_data/test__enc_lz4__idx_k_u/config.ini + +--echo # +--echo # ============================================ +--echo # Control: non-encrypted table keeps the selected compression on its data CF +--echo # ============================================ +--echo # +CREATE TABLE plain_lz4 ( + id INT PRIMARY KEY, + v VARCHAR(200) +) ENGINE=TIDESDB COMPRESSION='LZ4'; + +--echo # data CF: compression_algorithm = LZ4 (not encrypted, so compression stays) +--exec grep '^compression_algorithm' $MYSQLTEST_VARDIR/mysqld.1/tidesdb_data/test__plain_lz4/config.ini + +--echo # +--echo # ============================================ +--echo # Data still round-trips under an encrypted, compression-disabled CF +--echo # ============================================ +--echo # +INSERT INTO enc_lz4 VALUES (1, 10, 'alpha'), (2, 20, 'beta'), (3, 30, 'gamma'); +UPDATE enc_lz4 SET v = 'delta' WHERE id = 2; +DELETE FROM enc_lz4 WHERE id = 3; +SELECT * FROM enc_lz4 ORDER BY id; +--echo # covering index scan on the still-compressed index CF +SELECT id FROM enc_lz4 WHERE u = 10; + +--echo # +--echo # ============================================ +--echo # TRUNCATE recreates the data CF with compression still off +--echo # ============================================ +--echo # +TRUNCATE TABLE enc_lz4; +--exec grep '^compression_algorithm' $MYSQLTEST_VARDIR/mysqld.1/tidesdb_data/test__enc_lz4/config.ini +--exec grep '^compression_algorithm' $MYSQLTEST_VARDIR/mysqld.1/tidesdb_data/test__enc_lz4__idx_k_u/config.ini + +DROP TABLE enc_lz4; +DROP TABLE plain_lz4; + +--echo # +--source include/cleanup_tidesdb.inc +--echo # Done. diff --git a/mysql-test/suite/tidesdb/t/tidesdb_keyread_decodable.test b/mysql-test/suite/tidesdb/t/tidesdb_keyread_decodable.test new file mode 100644 index 00000000..6cb50312 --- /dev/null +++ b/mysql-test/suite/tidesdb/t/tidesdb_keyread_decodable.test @@ -0,0 +1,117 @@ +--source include/have_tidesdb.inc +--source include/not_embedded.inc + +# +# HA_KEYREAD_ONLY (index-only "Using index" plans) must only be advertised for +# indexes whose columns can be reconstructed from their mem-comparable sort key +# by decode_sort_key_part. For undecodable column types (VARCHAR, DECIMAL, +# floating point, multi-byte CHAR) a keyread would fall back to a primary-key +# row fetch at runtime, so advertising a covering plan mis-prices the query. +# +# We FORCE INDEX so the plan deterministically uses the secondary index, then +# read the Extra column: "Using index" appears only when the index-only read is +# genuinely available. The rows estimate is masked as it is not stable. +# + +--echo # +--echo # ============================================ +--echo # Decodable column types: covering (Using index) IS offered +--echo # ============================================ +--echo # +CREATE TABLE cov_ok ( + id INT PRIMARY KEY, + i INT, + b BIGINT, + d DATE, + ts DATETIME, + c CHAR(8) CHARACTER SET binary, + KEY k_i (i), + KEY k_b (b), + KEY k_d (d), + KEY k_ts (ts), + KEY k_c (c) +) ENGINE=TIDESDB; +INSERT INTO cov_ok VALUES + (1, 10, 100, '2020-01-01', '2020-01-01 10:00:00', 'aaa'), + (2, 20, 200, '2021-02-02', '2021-02-02 11:00:00', 'bbb'), + (3, 30, 300, '2022-03-03', '2022-03-03 12:00:00', 'ccc'); + +--replace_column 9 # +EXPLAIN SELECT i FROM cov_ok FORCE INDEX (k_i) WHERE i = 20; +--replace_column 9 # +EXPLAIN SELECT b FROM cov_ok FORCE INDEX (k_b) WHERE b = 200; +--replace_column 9 # +EXPLAIN SELECT d FROM cov_ok FORCE INDEX (k_d) WHERE d = '2021-02-02'; +--replace_column 9 # +EXPLAIN SELECT ts FROM cov_ok FORCE INDEX (k_ts) WHERE ts = '2021-02-02 11:00:00'; +--replace_column 9 # +EXPLAIN SELECT c FROM cov_ok FORCE INDEX (k_c) WHERE c = 'bbb'; + +--echo # +--echo # ============================================ +--echo # Undecodable column types: covering is NOT offered (no Using index) +--echo # ============================================ +--echo # +CREATE TABLE cov_no ( + id INT PRIMARY KEY, + s VARCHAR(50), + u CHAR(8) CHARACTER SET utf8mb4, + m DECIMAL(10,2), + f DOUBLE, + KEY k_s (s), + KEY k_u (u), + KEY k_m (m), + KEY k_f (f) +) ENGINE=TIDESDB; +INSERT INTO cov_no VALUES + (1, 'alpha', 'x1', 1.50, 1.5), + (2, 'beta', 'x2', 2.50, 2.5), + (3, 'gamma', 'x3', 3.50, 3.5); + +--replace_column 9 # +EXPLAIN SELECT s FROM cov_no FORCE INDEX (k_s) WHERE s = 'beta'; +--replace_column 9 # +EXPLAIN SELECT u FROM cov_no FORCE INDEX (k_u) WHERE u = 'x2'; +--replace_column 9 # +EXPLAIN SELECT m FROM cov_no FORCE INDEX (k_m) WHERE m = 2.50; +--replace_column 9 # +EXPLAIN SELECT f FROM cov_no FORCE INDEX (k_f) WHERE f = 2.5; + +--echo # results are still correct on the non-covering (PK fetch) path +SELECT s FROM cov_no FORCE INDEX (k_s) WHERE s = 'beta'; +SELECT m FROM cov_no FORCE INDEX (k_m) WHERE m = 2.50; + +--echo # +--echo # ============================================ +--echo # Composite index: covering decided per key part, not per whole index +--echo # ============================================ +--echo # +CREATE TABLE cov_mix ( + id INT PRIMARY KEY, + a INT, + b INT, + s VARCHAR(50), + KEY k_ab (a, b), + KEY k_as (a, s) +) ENGINE=TIDESDB; +INSERT INTO cov_mix VALUES (1, 1, 10, 'x'), (2, 2, 20, 'y'), (3, 3, 30, 'z'); + +--echo # (a INT, b INT): both decodable -> Using index +--replace_column 9 # +EXPLAIN SELECT a, b FROM cov_mix FORCE INDEX (k_ab) WHERE a = 2; +--echo # (a INT, s VARCHAR), reading only the decodable a -> still Using index +--replace_column 9 # +EXPLAIN SELECT a FROM cov_mix FORCE INDEX (k_as) WHERE a = 2; +--echo # (a INT, s VARCHAR), reading the undecodable s -> no Using index +--replace_column 9 # +EXPLAIN SELECT a, s FROM cov_mix FORCE INDEX (k_as) WHERE a = 2; +--echo # result on the s (non-covering) path is still correct +SELECT a, s FROM cov_mix FORCE INDEX (k_as) WHERE a = 2; + +DROP TABLE cov_ok; +DROP TABLE cov_no; +DROP TABLE cov_mix; + +--echo # +--source include/cleanup_tidesdb.inc +--echo # Done. diff --git a/mysql-test/suite/tidesdb/t/tidesdb_row_cardinality.test b/mysql-test/suite/tidesdb/t/tidesdb_row_cardinality.test new file mode 100644 index 00000000..d4290324 --- /dev/null +++ b/mysql-test/suite/tidesdb/t/tidesdb_row_cardinality.test @@ -0,0 +1,62 @@ +--source include/have_tidesdb.inc +--source include/not_embedded.inc + +# +# The row-count estimate (information_schema.TABLES.TABLE_ROWS, stats.records) +# is derived from tidesdb_cf_estimate_cardinality, which counts each key once +# per SSTable regardless of how many MVCC versions it holds and also counts the +# CF's entries in the (default) shared unified memtable. For a table written +# with only distinct keys and no updates every key exists exactly once, so the +# estimate is exact. Two independent tables confirm the count is attributed to +# each column family separately rather than summing the shared memtable. +# + +CREATE TABLE t_a (id INT PRIMARY KEY, v INT, KEY k_v (v)) ENGINE=TIDESDB; +CREATE TABLE t_b (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB; + +--disable_query_log +let $i = 1; +while ($i <= 200) +{ + eval INSERT INTO t_a VALUES ($i, $i MOD 10); + inc $i; +} +let $i = 1; +while ($i <= 75) +{ + eval INSERT INTO t_b VALUES ($i, $i); + inc $i; +} +--enable_query_log + +ANALYZE TABLE t_a; +ANALYZE TABLE t_b; + +--echo # each table reports its own distinct row count, not the shared-memtable sum +--echo # t_a -> 200 +SELECT TABLE_ROWS FROM information_schema.TABLES + WHERE table_schema = DATABASE() AND table_name = 't_a'; +--echo # t_b -> 75 +SELECT TABLE_ROWS FROM information_schema.TABLES + WHERE table_schema = DATABASE() AND table_name = 't_b'; + +--echo # inserting more rows raises the estimate on the next refresh +--disable_query_log +let $i = 201; +while ($i <= 300) +{ + eval INSERT INTO t_a VALUES ($i, $i MOD 10); + inc $i; +} +--enable_query_log +ANALYZE TABLE t_a; +--echo # t_a -> 300 +SELECT TABLE_ROWS FROM information_schema.TABLES + WHERE table_schema = DATABASE() AND table_name = 't_a'; + +DROP TABLE t_a; +DROP TABLE t_b; + +--echo # +--source include/cleanup_tidesdb.inc +--echo # Done. diff --git a/mysql-test/suite/tidesdb/t/tidesdb_secondary_gt_range.test b/mysql-test/suite/tidesdb/t/tidesdb_secondary_gt_range.test new file mode 100644 index 00000000..3aa7c0bf --- /dev/null +++ b/mysql-test/suite/tidesdb/t/tidesdb_secondary_gt_range.test @@ -0,0 +1,65 @@ +--source include/have_tidesdb.inc +--source include/not_embedded.inc + +# +# WHERE seccol > X on a secondary index (HA_READ_AFTER_KEY) must skip every +# entry equal to X and position on the first strictly-greater value. The engine +# now does this with a single upper-bound seek instead of stepping over each +# duplicate. These tests pin the behaviour, including the low-cardinality case +# (many rows share X) and the pathological row whose primary key encodes to the +# maximum key bytes and lands exactly on the upper bound. +# + +--echo # +--echo # ============================================ +--echo # > X excludes all X duplicates, includes only greater values +--echo # ============================================ +--echo # +CREATE TABLE t1 (id INT PRIMARY KEY, u INT, KEY k_u (u)) ENGINE=TIDESDB; +INSERT INTO t1 VALUES + (1, 2), (2, 2), (3, 2), (4, 2), (5, 2), + (6, 3), (7, 3), + (8, 4), + (9, 1), (10, 1); + +--echo # u > 2 -> only the u=3 and u=4 rows +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 2 ORDER BY u, id; +--echo # u >= 2 (control) -> includes the u=2 rows +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u >= 2 ORDER BY u, id; +--echo # u > 4 (greatest value) -> empty +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 4 ORDER BY u, id; +--echo # bounded range u > 2 AND u < 4 -> only u=3 +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 2 AND u < 4 ORDER BY id; + +--echo # +--echo # ============================================ +--echo # Pathological: a row whose PK encodes to the max key bytes (INT max) +--echo # sits at X's upper bound and must still be excluded by > X +--echo # ============================================ +--echo # +INSERT INTO t1 VALUES (2147483647, 2); +--echo # the max-PK row has u=2, so u > 2 must NOT return it +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u > 2 ORDER BY u, id; +--echo # u >= 2 must include it +SELECT id, u FROM t1 FORCE INDEX (k_u) WHERE u >= 2 AND u < 3 ORDER BY id; + +--echo # +--echo # ============================================ +--echo # Composite index: > on the leading part +--echo # ============================================ +--echo # +CREATE TABLE t2 (id INT PRIMARY KEY, a INT, b INT, KEY k_ab (a, b)) ENGINE=TIDESDB; +INSERT INTO t2 VALUES + (1, 1, 10), (2, 1, 20), (3, 1, 30), + (4, 2, 10), (5, 2, 20), + (6, 3, 99); + +--echo # a > 1 -> only a=2 and a=3 rows +SELECT id, a, b FROM t2 FORCE INDEX (k_ab) WHERE a > 1 ORDER BY a, b; + +DROP TABLE t1; +DROP TABLE t2; + +--echo # +--source include/cleanup_tidesdb.inc +--echo # Done. diff --git a/mysql-test/suite/tidesdb/t/tidesdb_stmt_atomicity.test b/mysql-test/suite/tidesdb/t/tidesdb_stmt_atomicity.test new file mode 100644 index 00000000..8a7268c8 --- /dev/null +++ b/mysql-test/suite/tidesdb/t/tidesdb_stmt_atomicity.test @@ -0,0 +1,152 @@ +--source include/have_tidesdb.inc +--source include/not_embedded.inc + +# +# Statement atomicity inside a multi-statement transaction. +# A statement that fails part-way (e.g. a multi-row INSERT whose later row +# duplicates a key) must roll back ONLY that statement, leaving earlier +# statements in the same BEGIN...COMMIT intact, and must not shift the +# transaction snapshot. Exercises the per-statement savepoint plus the +# revert of plugin-side shadow state (txn_key_state, unique-secondary and +# FTS bookkeeping). +# + +--echo # +--echo # ============================================ +--echo # TEST 1: failed statement leaves earlier ones intact (PRIMARY KEY) +--echo # ============================================ +--echo # +CREATE TABLE t1 (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB; + +START TRANSACTION; +INSERT INTO t1 VALUES (1, 10); +INSERT INTO t1 VALUES (2, 20); +--echo # row 3 inserts, then row 1 duplicates -> whole statement rolls back +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES (3, 30), (1, 99); +--echo # inside the txn: 1 and 2 survive, 3 is gone, 1 is unchanged (v=10) +SELECT * FROM t1 ORDER BY id; +COMMIT; +--echo # after commit: same result persisted +SELECT * FROM t1 ORDER BY id; + +--echo # +--echo # ============================================ +--echo # TEST 2: re-insert of a rolled-back key succeeds (txn_key_state revert) +--echo # ============================================ +--echo # +START TRANSACTION; +INSERT INTO t1 VALUES (5, 50); +--echo # row 6 inserts, then row 5 duplicates -> statement rolls back, 6 undone +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES (6, 60), (5, 999); +--echo # 6 was rolled back, so re-inserting it must NOT report a false duplicate +INSERT INTO t1 VALUES (6, 66); +COMMIT; +SELECT * FROM t1 ORDER BY id; + +--echo # +--echo # ============================================ +--echo # TEST 3: UNIQUE secondary index revert + re-insert +--echo # ============================================ +--echo # +CREATE TABLE t2 (id INT PRIMARY KEY, u INT UNIQUE, v INT) ENGINE=TIDESDB; + +START TRANSACTION; +INSERT INTO t2 VALUES (1, 100, 1); +--echo # row (2,200) inserts, then u=100 duplicates -> statement rolls back +--error ER_DUP_ENTRY +INSERT INTO t2 VALUES (2, 200, 2), (3, 100, 3); +--echo # the rolled-back unique value 200 must be free to insert again +INSERT INTO t2 VALUES (2, 200, 2); +COMMIT; +SELECT * FROM t2 ORDER BY id; + +--echo # +--echo # ============================================ +--echo # TEST 4: failed UPDATE leaves earlier UPDATE intact +--echo # ============================================ +--echo # +START TRANSACTION; +UPDATE t1 SET v = v + 1; +--echo # moving id 2 onto existing id 1 duplicates the PK -> statement rolls back +--error ER_DUP_ENTRY +UPDATE t1 SET id = 1 WHERE id = 2; +--echo # first UPDATE's increment survives; row 2 keeps its id +SELECT * FROM t1 ORDER BY id; +COMMIT; +SELECT * FROM t1 ORDER BY id; + +--echo # +--echo # ============================================ +--echo # TEST 5: statement rollback preserves the transaction snapshot +--echo # ============================================ +--echo # +CREATE TABLE t3 (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB; +INSERT INTO t3 VALUES (1, 1); + +connect (con1, localhost, root,,); +connection con1; +BEGIN; +--echo # pin con1's snapshot +SELECT * FROM t3 ORDER BY id; + +connection default; +INSERT INTO t3 VALUES (2, 2); + +connection con1; +--echo # a failed statement inside con1's txn must roll back only itself... +--error ER_DUP_ENTRY +INSERT INTO t3 VALUES (3, 3), (1, 3); +--echo # ...and must NOT shift the snapshot: con1 still must not see row 2 +SELECT * FROM t3 ORDER BY id; +COMMIT; +--echo # after commit the snapshot is released, row 2 becomes visible +SELECT * FROM t3 ORDER BY id; + +disconnect con1; +connection default; + +--echo # +--echo # ============================================ +--echo # TEST 6: FULLTEXT index stays consistent after a statement rollback +--echo # ============================================ +--echo # +CREATE TABLE ft (id INT PRIMARY KEY, body TEXT, FULLTEXT(body)) ENGINE=TIDESDB; + +START TRANSACTION; +INSERT INTO ft VALUES (1, 'alpha beta'); +--echo # row 2 (gamma delta) inserts, then id 1 duplicates -> statement rolls back +--error ER_DUP_ENTRY +INSERT INTO ft VALUES (2, 'gamma delta'), (1, 'dup'); +COMMIT; +--echo # gamma was rolled back -> no match; alpha (committed row 1) still matches +SELECT id FROM ft WHERE MATCH(body) AGAINST('gamma'); +SELECT id FROM ft WHERE MATCH(body) AGAINST('alpha'); +--echo # a fresh insert of the rolled-back term indexes cleanly +INSERT INTO ft VALUES (2, 'gamma delta'); +SELECT id FROM ft WHERE MATCH(body) AGAINST('gamma'); + +--echo # +--echo # ============================================ +--echo # TEST 7: user SAVEPOINT still works alongside statement savepoints +--echo # ============================================ +--echo # +START TRANSACTION; +INSERT INTO t1 VALUES (40, 400); +SAVEPOINT a; +INSERT INTO t1 VALUES (41, 410); +ROLLBACK TO SAVEPOINT a; +--echo # 40 survives, 41 undone by the user rollback +INSERT INTO t1 VALUES (42, 420); +COMMIT; +SELECT id FROM t1 WHERE id >= 40 ORDER BY id; + +DROP TABLE ft; +DROP TABLE t3; +DROP TABLE t2; +DROP TABLE t1; + +--echo # +--source include/cleanup_tidesdb.inc +--echo # Done. diff --git a/tidesdb/ha_tidesdb.cc b/tidesdb/ha_tidesdb.cc index 1b8dc3cf..aef55aa1 100644 --- a/tidesdb/ha_tidesdb.cc +++ b/tidesdb/ha_tidesdb.cc @@ -997,6 +997,40 @@ static int row_lock_acquire(tidesdb_trx_t *trx, const uchar *key, uint len, THD promotes any waiters now compatible with the remaining granted set, and broadcasts the lock's cond. Called from commit and rollback. */ +/* Release one granted request, promoting and waking any waiters it unblocks. + Caller owns removing req from its trx->held_locks_head chain. */ +static void tdb_lock_release_one(tdb_lock_request_t *req) +{ + tdb_row_lock_t *lock = req->lock; + uint part_idx = lock->partition; + tdb_lock_partition_t *part = &lock_partitions[part_idx]; + + mysql_mutex_lock(&part->mutex); + + /* Unlink req from lock->granted_head. */ + tdb_lock_request_t **pp = &lock->granted_head; + while (*pp && *pp != req) pp = &(*pp)->list_next; + if (*pp == req) *pp = req->list_next; + + /* Promote any waiters now grantable, then wake them up. */ + bool had_waiters = (lock->waiting_head != NULL); + tdb_lock_promote_waiters(lock); + bool promoted_any = had_waiters && (lock->granted_head != NULL); + + /* If nothing references this slot any more, unlink it from the + hash chain and stash it on the partition freelist so the next + acquire can reuse it without growing the chain. Slot memory + is retained across reuse for the deadlock walker. */ + tdb_lock_freelist_if_empty(part, lock); + + mysql_mutex_unlock(&part->mutex); + + if (had_waiters && (promoted_any || lock->waiting_head == NULL)) + mysql_cond_broadcast(&lock->cond); + + my_free(req); +} + static void row_locks_release_all(tidesdb_trx_t *trx) { if (!lock_partitions || !trx) return; @@ -1006,40 +1040,134 @@ static void row_locks_release_all(tidesdb_trx_t *trx) while (req) { tdb_lock_request_t *next = req->held_next; - tdb_row_lock_t *lock = req->lock; - uint part_idx = lock->partition; - tdb_lock_partition_t *part = &lock_partitions[part_idx]; + tdb_lock_release_one(req); + released++; + req = next; + } + trx->held_locks_head = NULL; + trx->waiting_on_lock.store(NULL, std::memory_order_relaxed); + if (released > 0) srv_stat_lock_held.fetch_sub(released, std::memory_order_relaxed); +} - mysql_mutex_lock(&part->mutex); +/* Release only the locks acquired since a statement-start marker, leaving the + marker and everything below it (locks from earlier statements) held. Because + row_lock_acquire pushes new grants onto the head and re-entry never adds a + duplicate request, the statement's locks are exactly the run from the current + head down to (but excluding) marker. */ +static void row_locks_release_since(tidesdb_trx_t *trx, tdb_lock_request_t *marker) +{ + if (!lock_partitions || !trx) return; - /* Unlink req from lock->granted_head. */ - tdb_lock_request_t **pp = &lock->granted_head; - while (*pp && *pp != req) pp = &(*pp)->list_next; - if (*pp == req) *pp = req->list_next; + long long released = 0; + tdb_lock_request_t *req = trx->held_locks_head; + while (req && req != marker) + { + tdb_lock_request_t *next = req->held_next; + tdb_lock_release_one(req); + released++; + req = next; + } + trx->held_locks_head = marker; + if (released > 0) srv_stat_lock_held.fetch_sub(released, std::memory_order_relaxed); +} - /* Promote any waiters now grantable, then wake them up. */ - bool had_waiters = (lock->waiting_head != NULL); - tdb_lock_promote_waiters(lock); - bool promoted_any = had_waiters && (lock->granted_head != NULL); +/* Reserved name of the per-statement savepoint. The SQL SAVEPOINT callbacks + synthesize "sv_%p" names, so this never collides with a user savepoint. */ +static constexpr const char TIDESDB_STMT_SAVEPOINT[] = "stmt"; - /* If nothing references this slot any more, unlink it from the - hash chain and stash it on the partition freelist so the next - acquire can reuse it without growing the chain. Slot memory - is retained across reuse for the deadlock walker. */ - tdb_lock_freelist_if_empty(part, lock); +/* Set trx->txn_key_state[key] = val, journaling the prior value while a + statement savepoint is armed so a statement rollback can restore it. */ +static inline void trx_set_key_state(tidesdb_trx_t *trx, const std::string &key, bool val) +{ + if (trx->stmt_savepoint_active) + { + auto it = trx->txn_key_state.find(key); + signed char prior = (it == trx->txn_key_state.end()) ? -1 : (it->second ? 1 : 0); + trx->stmt_key_state_undo.push_back({key, prior}); + } + trx->txn_key_state[key] = val; +} - mysql_mutex_unlock(&part->mutex); +/* Arm a statement savepoint at statement start. Caller guarantees we are in a + multi-statement transaction (not autocommit/DDL), trx->txn exists, and no + statement savepoint is currently armed. On any failure we leave the savepoint + unarmed so a later statement rollback safely falls back to full rollback. */ +static void stmt_savepoint_arm(tidesdb_trx_t *trx) +{ + /* Defensive: a prior statement's savepoint should already be gone (released + on success, removed on rollback), but a user ROLLBACK TO SAVEPOINT can + collaterally drop it; releasing again is a cheap no-op if absent. */ + (void)tidesdb_txn_release_savepoint(trx->txn, TIDESDB_STMT_SAVEPOINT); + if (tidesdb_txn_savepoint(trx->txn, TIDESDB_STMT_SAVEPOINT) != TDB_SUCCESS) return; - if (had_waiters && (promoted_any || lock->waiting_head == NULL)) - mysql_cond_broadcast(&lock->cond); + trx->stmt_savepoint_active = true; + trx->stmt_lock_marker = trx->held_locks_head; + trx->stmt_key_state_undo.clear(); + trx->stmt_fts_snapshot = trx->fts_meta_pending; + trx->stmt_fts_dirty_snapshot = trx->fts_meta_dirty; +} - my_free(req); - released++; - req = next; +/* Statement completed successfully -- the statement's writes are now permanent + within the (still uncommitted) txn. Drop the savepoint and per-statement undo + state so the next statement re-arms cleanly. */ +static void stmt_savepoint_disarm(tidesdb_trx_t *trx) +{ + if (!trx->stmt_savepoint_active) return; + (void)tidesdb_txn_release_savepoint(trx->txn, TIDESDB_STMT_SAVEPOINT); + trx->stmt_savepoint_active = false; + trx->stmt_lock_marker = nullptr; + trx->stmt_key_state_undo.clear(); + trx->stmt_fts_snapshot.clear(); +} + +/* Roll back just the current statement's effects to the armed savepoint. + Returns true if the partial rollback was performed, false if the caller must + fall back to a full transaction rollback (no savepoint armed, or the library + savepoint is gone because a bulk mid-commit reset the txn or a user ROLLBACK + TO SAVEPOINT removed it). */ +static bool stmt_savepoint_rollback(tidesdb_trx_t *trx) +{ + if (!trx->stmt_savepoint_active) return false; + + int rc = tidesdb_txn_rollback_to_savepoint(trx->txn, TIDESDB_STMT_SAVEPOINT); + if (rc != TDB_SUCCESS) + { + /* Savepoint vanished under us -- cannot do a partial rollback. */ + trx->stmt_savepoint_active = false; + trx->stmt_lock_marker = nullptr; + trx->stmt_key_state_undo.clear(); + trx->stmt_fts_snapshot.clear(); + return false; } - trx->held_locks_head = NULL; - trx->waiting_on_lock.store(NULL, std::memory_order_relaxed); - if (released > 0) srv_stat_lock_held.fetch_sub(released, std::memory_order_relaxed); + + /* Revert the plugin-side shadow state to the same boundary. Undo the + txn_key_state mutations in reverse so a key set twice restores correctly. */ + for (auto it = trx->stmt_key_state_undo.rbegin(); it != trx->stmt_key_state_undo.rend(); ++it) + { + if (it->second < 0) + trx->txn_key_state.erase(it->first); + else + trx->txn_key_state[it->first] = (it->second == 1); + } + trx->stmt_key_state_undo.clear(); + + trx->fts_meta_pending = trx->stmt_fts_snapshot; + trx->fts_meta_dirty = trx->stmt_fts_dirty_snapshot; + trx->stmt_fts_snapshot.clear(); + + /* Drop only the locks the failed statement took; earlier statements keep + theirs. An S->X upgrade on an earlier statement's lock stays X, which is + conservative but never incorrect. */ + row_locks_release_since(trx, trx->stmt_lock_marker); + trx->stmt_lock_marker = nullptr; + + /* The library freed the txn ops appended after the savepoint, so any cached + iterator over this txn (scan_iter, dup_iter_cache_ on every handler on + this connection) now has a stale view. Bumping the generation makes each + handler rebuild lazily, exactly as a bulk mid-commit does. */ + trx->txn_generation++; + trx->stmt_savepoint_active = false; + return true; } /* Pick the lock mode for a row materialised on a read path, or report @@ -3528,6 +3656,21 @@ static tidesdb_column_family_config_t build_cf_config(const ha_table_option_stru return cfg; } +/* Rows in a table's data column family are encrypted in serialize_row before + they reach the library, and ciphertext does not compress, so running the + block and value-log compressor over it on every flush and compaction spends + CPU for no space saving. Return a copy of the config with compression forced + off when the table is encrypted, leaving every other setting untouched. This + is for the data CF only -- secondary-index CFs hold unencrypted comparable + keys and keep whatever compression the table selected. */ +static tidesdb_column_family_config_t data_cf_config(const tidesdb_column_family_config_t &cfg, + bool encrypted) +{ + tidesdb_column_family_config_t data = cfg; + if (encrypted) data.compression_algorithm = (compression_algorithm)TDB_COMPRESS_NONE; + return data; +} + /* Resolve a secondary index CF by name. Returns the CF pointer (may be NULL if not found). @@ -3755,21 +3898,13 @@ static int tidesdb_commit(handlerton *, THD *thd, bool all) { /* Statement-level commit inside a multi-statement transaction. Defer the actual commit -- writes stay buffered in the txn, - avoiding expensive txn_begin + commit per statement. - - tidesdb_txn_savepoint() deep-copies the entire - write-set (malloc+memcpy for every key/value). For a txn - with N ops across S statements, total copy cost is - O(S * N * avg_kv_size) -- quadratic and devastating for - multi-statement OLTP transactions. - - We skip the per-statement savepoint entirely. This means - statement-level rollback inside BEGIN...COMMIT falls back to - full transaction rollback (same as many simple SE's). - The trade-off is a statement failure aborts the entire txn - instead of undoing just that statement. For OLTP this is - acceptable since the client will retry the whole transaction - anyway after a conflict/error. */ + avoiding an expensive txn_begin + commit per statement. The + statement succeeded, so its writes become permanent within the + still-open txn; drop the statement savepoint armed in external_lock + so the next statement re-arms at the new boundary. (tidesdb_txn + savepoints are O(1) -- they record op/cf counts, they do not copy + the write-set -- so this per-statement arm/disarm is cheap.) */ + stmt_savepoint_disarm(trx); return 0; } @@ -3779,12 +3914,10 @@ static int tidesdb_commit(handlerton *, THD *thd, bool all) trx->txn_key_state.clear(); /* We must release any active statement savepoint before final commit/rollback. - Savepoints must be explicitly released before txn_commit. */ - if (trx->stmt_savepoint_active) - { - tidesdb_txn_release_savepoint(trx->txn, "stmt"); - trx->stmt_savepoint_active = false; - } + Savepoints must be explicitly released before txn_commit. Disarm also + clears the per-statement undo journal and fts snapshot; it leaves + fts_meta_pending intact for the flush below. */ + stmt_savepoint_disarm(trx); /* Real commit -- flush to storage. After a successful commit, we keep the txn object alive and let @@ -3867,15 +4000,25 @@ static int tidesdb_rollback(handlerton *, THD *thd, bool all) if (!is_real_rollback) { /* Statement-level rollback inside a multi-statement transaction. - Without per-statement savepoints (see tidesdb_commit note), - we fall through to full transaction rollback. This is the - same behavior as many simple storage engines and is correct -- - OLTP clients retry the entire transaction after any error. */ + Roll back only this statement's effects to the savepoint armed in + external_lock, preserving the rest of the transaction and its + snapshot. stmt_savepoint_rollback reverts the library op array and + the plugin-side shadow state (txn_key_state, fts_meta_pending, and + the statement's row locks) together. It returns false when a partial + rollback is impossible -- no savepoint armed, or a bulk mid-commit or + user ROLLBACK TO SAVEPOINT removed it -- and we fall through to a full + transaction rollback in that case. */ + if (stmt_savepoint_rollback(trx)) + { + /* Leave trx->dirty as-is: earlier statements' writes remain in the + still-open txn, and its snapshot is deliberately preserved. */ + return 0; + } } if (trx->stmt_savepoint_active) { - tidesdb_txn_release_savepoint(trx->txn, "stmt"); + tidesdb_txn_release_savepoint(trx->txn, TIDESDB_STMT_SAVEPOINT); trx->stmt_savepoint_active = false; } @@ -3884,6 +4027,9 @@ static int tidesdb_rollback(handlerton *, THD *thd, bool all) trx->fts_meta_pending.clear(); trx->fts_meta_dirty = false; trx->txn_key_state.clear(); + trx->stmt_key_state_undo.clear(); + trx->stmt_fts_snapshot.clear(); + trx->stmt_lock_marker = nullptr; /* Full rollback -- we keep txn alive for reuse via reset on next use. */ tidesdb_txn_rollback(trx->txn); @@ -3910,7 +4056,6 @@ static int tidesdb_close_connection(handlerton *, THD *thd) tidesdb_txn_rollback(trx->txn); tidesdb_txn_free(trx->txn); } - if (trx->dup_rtxn) tidesdb_txn_free(trx->dup_rtxn); delete trx; thd_set_ha_data(thd, tidesdb_hton, NULL); } @@ -5567,8 +5712,64 @@ bool ha_tidesdb::decode_int_sort_key(const uint8_t *src, uint sort_len, bool is_ Returns true on success, false for unsupported types. */ + +/* Whether decode_sort_key_part can reconstruct this field's value from its + comparable sort key. Depends only on the field type and charset, not on any + stored bytes, so index_flags can use it to decide index-only capability + without touching data. This is the single source of truth for the set of + decodable types -- decode_sort_key_part checks it first, so the two never + drift. The mem-comparable sort key is only invertible for these types; for + VARCHAR, DECIMAL, floating point, multi-byte CHAR, BLOB, ENUM/SET and the + like the sort weights are lossy, so a keyread of such a column must fall back + to the primary-key row fetch. */ +static bool is_keyread_decodable_type(const Field *f) +{ + switch (f->real_type()) + { + case MYSQL_TYPE_TINY: + case MYSQL_TYPE_SHORT: + case MYSQL_TYPE_INT24: + case MYSQL_TYPE_LONG: + case MYSQL_TYPE_LONGLONG: + case MYSQL_TYPE_YEAR: + case MYSQL_TYPE_DATE: + case MYSQL_TYPE_NEWDATE: + case MYSQL_TYPE_DATETIME: + case MYSQL_TYPE_DATETIME2: + case MYSQL_TYPE_TIMESTAMP: + case MYSQL_TYPE_TIMESTAMP2: + return true; + case MYSQL_TYPE_STRING: + /* Fixed CHAR/BINARY only for charsets whose sort weights equal the + stored bytes. */ + return f->charset() == &my_charset_bin || f->charset() == &my_charset_latin1; + default: + return false; + } +} + +/* Whether the field at key part `part` of index `keyno` reconstructs from its + sort key. Resolved through key_part.fieldnr and table_share->field rather + than key_part.field, because index_flags builds field->part_of_key on a + fresh, unopened handler at frm-parse time where key_part.field and the plugin + share are not yet available. An unresolvable field is treated conservatively + as undecodable. */ +static bool index_part_is_decodable(const TABLE_SHARE *ts, uint keyno, uint part) +{ + if (!ts || !ts->field || keyno >= ts->keys) return false; + const KEY *k = &ts->key_info[keyno]; + if (part >= k->user_defined_key_parts) return false; + uint fnr = k->key_part[part].fieldnr; + if (fnr == 0 || fnr - 1 >= ts->fields) return false; + return is_keyread_decodable_type(ts->field[fnr - 1]); +} + bool ha_tidesdb::decode_sort_key_part(const uint8_t *src, uint sort_len, Field *f, uchar *buf) { + /* Reject undecodable types up front so this stays in lockstep with + is_keyread_decodable_type, which index_flags relies on. */ + if (!is_keyread_decodable_type(f)) return false; + /* Compute the destination pointer exactly once per call. Every branch below wrote `buf + (f->ptr - f->table->record[0])` independently. */ uchar *to = buf + (uintptr_t)(f->ptr - f->table->record[0]); @@ -6146,11 +6347,12 @@ int ha_tidesdb::create(const char *name, TABLE *table_arg, HA_CREATE_INFO *creat } tidesdb_column_family_config_t cfg = build_cf_config(opts); + tidesdb_column_family_config_t data_cfg = data_cf_config(cfg, opts && opts->encrypted); /* We create main data CF (we simply skip if it already exists, e.g. crash recovery) */ if (!tidesdb_get_column_family(tdb_global, cf_name.c_str())) { - int rc = tidesdb_create_column_family(tdb_global, cf_name.c_str(), &cfg); + int rc = tidesdb_create_column_family(tdb_global, cf_name.c_str(), &data_cfg); if (rc != TDB_SUCCESS) { sql_print_error("[TIDESDB] Failed to create CF '%s' (err=%d)", cf_name.c_str(), rc); @@ -6709,21 +6911,20 @@ int ha_tidesdb::iter_read_current(uchar *buf) /* Probe whether a PK already exists for an INSERT/UPDATE uniqueness check - WITHOUT recording the read in the write transaction. - - A tracking tidesdb_txn_get() of an absent key records read-seq 0 in the txn - read-set; the library's first-committer-wins reservation then uses that 0 as - the write's conflict base, so a stale reservation slot left by any prior write - of the same key (slots survive across DROP/recreate) trips a spurious - TDB_ERR_CONFLICT under load. The probe must therefore stay out of the write - txn's read-set. We answer from two sources: - - -- same-txn pending state (trx->txn_key_state) -- this txn may have already - inserted (-> duplicate) or deleted (-> not a duplicate) the key, which - committed data alone cannot tell us; and - -- committed data, read on a dedicated READ_COMMITTED txn that is reset per - probe. Being read-only it never reserves writes (no base pollution), and - READ_COMMITTED is skipped by the snapshot floor so it never pins min_snap. + without recording the read into the write transaction's conflict footprint. + + tidesdb_txn_contains resolves at the write txn's snapshot but does not track + the read, so an absent-key probe cannot seed read-seq 0 into the first- + committer-wins reservation base -- a tracking get would, and a stale + reservation slot left by a prior write of the same key would then trip a + spurious TDB_ERR_CONFLICT under load. Uniqueness against a writer that + commits the same key after our snapshot is still enforced by this insert's + own reservation at commit time. We answer from two sources: + + -- same-txn pending state (trx->txn_key_state), which records a key this txn + has already inserted (-> duplicate) or deleted (-> not a duplicate), and + which committed data alone cannot reveal; and + -- a non-tracking existence check on the write txn for everything else. */ int ha_tidesdb::probe_pk_exists(tidesdb_trx_t *trx, const std::string &tkey, const uchar *dk, uint dk_len) @@ -6732,48 +6933,13 @@ int ha_tidesdb::probe_pk_exists(tidesdb_trx_t *trx, const std::string &tkey, con { auto it = trx->txn_key_state.find(tkey); if (it != trx->txn_key_state.end()) return it->second ? 1 : 0; - - int rc; - if (!trx->dup_rtxn) - { - rc = tidesdb_txn_begin_with_isolation(tdb_global, TDB_ISOLATION_READ_COMMITTED, - &trx->dup_rtxn); - } - else - { - /* tidesdb_txn_reset requires a committed txn (it is the post-commit - reuse path). Finalize the previous probe's read-only txn first -- - num_ops==0 at READ_COMMITTED takes the library's read-only commit - fast path, so this is cheap and never reserves anything. */ - tidesdb_txn_commit(trx->dup_rtxn); - rc = tidesdb_txn_reset(trx->dup_rtxn, TDB_ISOLATION_READ_COMMITTED); - } - if (rc != TDB_SUCCESS) return -tdb_rc_to_ha(rc, "probe_pk_exists txn"); - - uint8_t *v = NULL; - size_t l = 0; - int g = tidesdb_txn_get(trx->dup_rtxn, share->cf, dk, dk_len, &v, &l); - if (g == TDB_SUCCESS) - { - if (v) tidesdb_free(v); - return 1; - } - if (g == TDB_ERR_NOT_FOUND) return 0; - return -tdb_rc_to_ha(g, "probe_pk_exists get"); } - /* No per-connection trx (rare). Fall back to the tracking get; without a - reused write txn there is no reservation base to pollute. */ - uint8_t *v = NULL; - size_t l = 0; - int g = tidesdb_txn_get(stmt_txn, share->cf, dk, dk_len, &v, &l); - if (g == TDB_SUCCESS) - { - if (v) tidesdb_free(v); - return 1; - } + tidesdb_txn_t *probe_txn = (trx && trx->txn) ? trx->txn : stmt_txn; + int g = tidesdb_txn_contains(probe_txn, share->cf, dk, dk_len); + if (g == TDB_SUCCESS) return 1; if (g == TDB_ERR_NOT_FOUND) return 0; - return -tdb_rc_to_ha(g, "probe_pk_exists get(fallback)"); + return -tdb_rc_to_ha(g, "probe_pk_exists contains"); } /* ******************** write_row (INSERT) ******************** */ @@ -6919,7 +7085,7 @@ int ha_tidesdb::write_row(const uchar *buf) /* Not a duplicate -- record the pending insert so a later insert of the same key in this same txn is still caught (committed data cannot show a pending insert). The put below stays blind, so base = snapshot. */ - if (trx) trx->txn_key_state[tkey] = true; + if (trx) trx_set_key_state(trx, tkey, true); } /* We check UNIQUE secondary index uniqueness. This honours the @@ -7566,15 +7732,26 @@ int ha_tidesdb::index_read_map(uchar *buf, const uchar *key, key_part_map keypar } else if (find_flag == HA_READ_AFTER_KEY) { - /* We seek, then skip past any exact prefix matches */ - tidesdb_iter_seek(scan_iter, comp_key, comp_len); - while (tidesdb_iter_valid(scan_iter)) + /* Skip every entry sharing this index-column prefix in one seek + instead of stepping over them one at a time. Secondary entries + are [comparable idx cols][pk], so the greatest entry with this + prefix is comp_key followed by the maximum pk (all 0xFF), the same + upper bound the PREV branch below builds. Seeking to it lands on + the first strictly-greater index value -- the only prefix match it + can land on is the pathological row whose pk itself encodes to all + 0xFF, which we then step past. */ + uchar upper[SEC_IDX_KEY_BUF_LEN]; + memcpy(upper, comp_key, comp_len); + memset(upper + comp_len, KEY_INF_HI_BYTE, share->pk_key_len); + uint upper_len = comp_len + share->pk_key_len; + tidesdb_iter_seek(scan_iter, upper, upper_len); + if (tidesdb_iter_valid(scan_iter)) { uint8_t *ik = NULL; size_t iks = 0; - if (tidesdb_iter_key(scan_iter, &ik, &iks) != TDB_SUCCESS) break; - if (iks < comp_len || memcmp(ik, comp_key, comp_len) != 0) break; - tidesdb_iter_next(scan_iter); + if (tidesdb_iter_key(scan_iter, &ik, &iks) == TDB_SUCCESS && iks >= comp_len && + memcmp(ik, comp_key, comp_len) == 0) + tidesdb_iter_next(scan_iter); } } else if (find_flag == HA_READ_KEY_OR_PREV || find_flag == HA_READ_BEFORE_KEY || @@ -8106,14 +8283,20 @@ int ha_tidesdb::update_row(const uchar *old_data, const uchar *new_data) uint old_dk_chk_len = build_data_key(old_pk, old_pk_len, old_dk_chk); std::string otkey(share->cf_name); otkey.append((const char *)old_dk_chk, old_dk_chk_len); - trx->txn_key_state[otkey] = false; - trx->txn_key_state[ntkey] = true; + trx_set_key_state(trx, otkey, false); + trx_set_key_state(trx, ntkey, true); } } if (share->num_secondary_indexes > 0) { const my_ptrdiff_t nd_ptrdiff = (my_ptrdiff_t)(new_data - table->record[0]); + /* Reuse the per-index dup-check iterators that write_row caches, + invalidating on txn change, so a multi-row UPDATE that shifts a + UNIQUE value does not rebuild the catastrophically expensive + tidesdb_iter_new() (O(num_sstables) merge-heap construction) on + every row. Mirrors the caching in write_row. */ + const uint64_t cur_gen = trx ? trx->txn_generation : 0; for (uint i = 0; i < table->s->keys; i++) { if (share->has_user_pk && i == share->pk_index) continue; @@ -8157,12 +8340,29 @@ int ha_tidesdb::update_row(const uchar *old_data, const uchar *new_data) memcmp(old_prefix, new_prefix, new_prefix_len) == 0) continue; - tidesdb_iter_t *dup_iter = NULL; - int irc = tdb_iter_new_blocking(ha_thd(), txn, share->idx_cfs[i], &dup_iter); - if (irc != TDB_SUCCESS || !dup_iter) + /* Get or create the cached dup-check iterator for this index, + invalidating if the txn changed (commit/reset frees the txn + ops the iterator's MERGE_SOURCE_TXN_OPS depends on). Shared + with write_row via dup_iter_cache_. */ + tidesdb_iter_t *dup_iter = dup_iter_cache_[i]; + if (dup_iter && (dup_iter_txn_[i] != txn || dup_iter_txn_gen_[i] != cur_gen)) { - tmp_restore_column_map(&table->read_set, old_map); - DBUG_RETURN(tdb_rc_to_ha(irc, "update_row dup_iter_new")); + tidesdb_iter_free(dup_iter); + dup_iter = NULL; + dup_iter_cache_[i] = NULL; + } + if (!dup_iter) + { + int irc = tdb_iter_new_blocking(ha_thd(), txn, share->idx_cfs[i], &dup_iter); + if (irc != TDB_SUCCESS || !dup_iter) + { + tmp_restore_column_map(&table->read_set, old_map); + DBUG_RETURN(tdb_rc_to_ha(irc, "update_row dup_iter_new")); + } + dup_iter_cache_[i] = dup_iter; + dup_iter_txn_[i] = txn; + dup_iter_txn_gen_[i] = cur_gen; + dup_iter_count_++; } tidesdb_iter_seek(dup_iter, new_prefix, new_prefix_len); @@ -8180,7 +8380,7 @@ int ha_tidesdb::update_row(const uchar *old_data, const uchar *new_data) memcpy(dup_ref, fk + new_prefix_len, suffix_len); } } - tidesdb_iter_free(dup_iter); + /* Cached iterator is retained for the next row; not freed here. */ if (dup) { @@ -8563,7 +8763,7 @@ int ha_tidesdb::delete_row(const uchar *buf) { std::string tkey(share->cf_name); tkey.append((const char *)dk, dk_len); - trx->txn_key_state[tkey] = false; + trx_set_key_state(trx, tkey, false); } /* We delete secondary index entries in a single consolidated dispatch loop. @@ -8698,7 +8898,9 @@ int ha_tidesdb::delete_all_rows(void) stmt_txn_dirty = false; } - tidesdb_column_family_config_t cfg = build_cf_config(TDB_TABLE_OPTIONS(table)); + const ha_table_option_struct *t_opts = TDB_TABLE_OPTIONS(table); + tidesdb_column_family_config_t cfg = build_cf_config(t_opts); + tidesdb_column_family_config_t data_cfg = data_cf_config(cfg, t_opts && t_opts->encrypted); { std::string cf_name = share->cf_name; @@ -8710,7 +8912,7 @@ int ha_tidesdb::delete_all_rows(void) DBUG_RETURN(tdb_rc_to_ha(rc, "truncate drop_cf")); } - rc = tidesdb_create_column_family(tdb_global, cf_name.c_str(), &cfg); + rc = tidesdb_create_column_family(tdb_global, cf_name.c_str(), &data_cfg); if (rc != TDB_SUCCESS) { sql_print_error("[TIDESDB] truncate: failed to recreate CF '%s' (err=%d)", @@ -8763,9 +8965,10 @@ int ha_tidesdb::delete_all_rows(void) /* Commit the current txn mid-statement and reset it with READ_COMMITTED so the next batch starts fresh. Shared by bulk INSERT/UPDATE/DELETE once - buffered ops cross TIDESDB_BULK_INSERT_BATCH_OPS -- keeps us under - TDB_MAX_TXN_OPS and bounds txn memory. Higher isolation levels would - cause unbounded read-set growth across batches. + buffered ops cross TIDESDB_BULK_INSERT_BATCH_OPS, which bounds the memory the + txn holds before commit since the whole write set lives in the ops array + until then. The reset drops to READ_COMMITTED so the read set does not grow + across batches the way a higher isolation level would. Any cached iterators and dup-check iterators are invalidated, they hold references to MERGE_SOURCE_TXN_OPS that txn_reset clears. @@ -8821,6 +9024,13 @@ int ha_tidesdb::maybe_bulk_commit(tidesdb_trx_t *trx) scan_iter_txn_ = NULL; } free_dup_iter_cache(); + /* The txn was torn down, so the armed statement savepoint and its lock + marker are void; disarm so a later statement rollback falls back to a + full rollback rather than a stale partial one. */ + trx->stmt_savepoint_active = false; + trx->stmt_lock_marker = nullptr; + trx->stmt_key_state_undo.clear(); + trx->stmt_fts_snapshot.clear(); return tdb_rc_to_ha(crc, "bulk_commit"); } @@ -8857,6 +9067,14 @@ int ha_tidesdb::maybe_bulk_commit(tidesdb_trx_t *trx) } free_dup_iter_cache(); scan_txn = trx->txn; + /* The mid-statement commit reset the txn and released every row lock, so the + armed statement savepoint and its lock marker no longer mean anything. + Disarm it -- the remainder of this statement can no longer be rolled back + atomically (the committed rows are durable), matching prior behavior. */ + trx->stmt_savepoint_active = false; + trx->stmt_lock_marker = nullptr; + trx->stmt_key_state_undo.clear(); + trx->stmt_fts_snapshot.clear(); return 0; } @@ -9166,7 +9384,20 @@ int ha_tidesdb::info(uint flag) tidesdb_stats_t *st = NULL; if (tidesdb_get_stats(share->cf, &st) == TDB_SUCCESS && st) { - share->cached_records.store(st->total_keys, std::memory_order_relaxed); + /* Prefer the distinct-key cardinality estimate for the row + count. Within an SSTable it counts each key once no matter + how many MVCC versions it holds, which total_keys counts + separately, so it is a tighter row estimate for the optimizer + (it still upper-bounds a key spread across several SSTables). + Fall back to total_keys when the library cannot supply it. */ + uint64_t rows = st->total_keys; + uint64_t est = 0; + /* The aggregate is always a real sum -- the per-SSTable + TDB_DISTINCT_KEYS_UNKNOWN sentinel is resolved to that + SSTable's entry count inside the library -- so success alone + means the value is usable. */ + if (tidesdb_cf_estimate_cardinality(share->cf, &est) == TDB_SUCCESS) rows = est; + share->cached_records.store(rows, std::memory_order_relaxed); /* total_data_size only counts SSTable klog+vlog; memtable_size holds the active memtable footprint. Sum both so that @@ -9885,7 +10116,18 @@ ulong ha_tidesdb::index_flags(uint idx, uint part, bool all_parts) const if (table_share && table_share->primary_key != MAX_KEY && idx == table_share->primary_key) flags |= HA_CLUSTERED_INDEX; else - flags |= HA_KEYREAD_ONLY; + { + /* The server builds field->part_of_key (the covering-index bitmap) by + calling index_flags(idx, part, 0) once per key part, so advertise + HA_KEYREAD_ONLY for a part only when that part's field reconstructs + from its sort key via decode_sort_key_part. A query that reads only + decodable columns of a mixed-type index still gets an index-only + plan; a read of an undecodable column (VARCHAR, DECIMAL, floating + point, multi-byte CHAR) leaves part_of_key clear so the optimizer + prices the primary-key row fetch instead of a phantom covering scan + that try_keyread_from_index would fall back on at runtime. */ + if (index_part_is_decodable(table_share, idx, part)) flags |= HA_KEYREAD_ONLY; + } return flags; } @@ -10525,6 +10767,16 @@ int ha_tidesdb::external_lock(THD *thd, int lock_type) trans_register_ha(thd, false, ht, 0); if (!is_autocommit) trans_register_ha(thd, true, ht, 0); + + /* Arm a statement savepoint so a statement error inside BEGIN...COMMIT + rolls back only this statement, not the whole transaction. Only for + real multi-statement transactions -- autocommit and DDL statements + are their own transaction, where statement rollback is already a full + rollback. external_lock fires once per table per statement, so the + !stmt_savepoint_active guard arms it exactly once, at the first table, + before any of the statement's row writes. */ + if (!is_autocommit && !is_ddl && trx->txn && !trx->stmt_savepoint_active) + stmt_savepoint_arm(trx); } else { @@ -11169,6 +11421,7 @@ bool ha_tidesdb::commit_inplace_alter_table(TABLE *altered_table, Alter_inplace_ } } } + share->num_secondary_indexes = 0; for (uint i = 0; i < share->idx_cfs.size(); i++) if (share->idx_cfs[i]) share->num_secondary_indexes++; @@ -11178,12 +11431,14 @@ bool ha_tidesdb::commit_inplace_alter_table(TABLE *altered_table, Alter_inplace_ of only being persisted in the .frm. */ if (ha_alter_info->handler_flags & ALTER_CHANGE_CREATE_OPTION) { - tidesdb_column_family_config_t cfg = build_cf_config(TDB_TABLE_OPTIONS(altered_table)); + const ha_table_option_struct *a_opts = TDB_TABLE_OPTIONS(altered_table); + tidesdb_column_family_config_t cfg = build_cf_config(a_opts); + tidesdb_column_family_config_t data_cfg = data_cf_config(cfg, a_opts && a_opts->encrypted); /* Main data CF */ if (share->cf) { - int rc = tidesdb_cf_update_runtime_config(share->cf, &cfg, 1); + int rc = tidesdb_cf_update_runtime_config(share->cf, &data_cfg, 1); if (rc != TDB_SUCCESS) sql_print_warning( "[TIDESDB] ALTER: failed to update runtime config for " @@ -11557,8 +11812,8 @@ static long long srv_stat_compaction_count; tidesdb_show_status can read them directly. Their definitions live up there. */ -#define TIDESQL_VERSION_STR "4.5.8" -#define TIDESQL_VERSION_HEX 0x40508 +#define TIDESQL_VERSION_STR "4.5.9" +#define TIDESQL_VERSION_HEX 0x40509 static const char *srv_stat_version = TIDESQL_VERSION_STR; static long long srv_stat_version_hex = TIDESQL_VERSION_HEX; diff --git a/tidesdb/ha_tidesdb.h b/tidesdb/ha_tidesdb.h index 8b58cc0f..4caf7471 100644 --- a/tidesdb/ha_tidesdb.h +++ b/tidesdb/ha_tidesdb.h @@ -564,17 +564,31 @@ struct tidesdb_trx_t std::vector fts_meta_pending; bool fts_meta_dirty{false}; - /* Uniqueness probing must not seed the WRITE txn's read-set, a point-get - of an absent key records read-seq 0, which the library's first-committer - reservation then uses as the conflict base, so a stale slot from any - prior write of that key trips a spurious conflict under load. Instead we - probe committed data on a dedicated READ_COMMITTED txn (no read-set, never - pins the snapshot floor) and track same-txn pending existence here: + /* Same-txn pending key existence, consulted by probe_pk_exists before its + non-tracking tidesdb_txn_contains check on the write txn. Committed data + alone cannot show what this txn has staged, so we record it here: true = inserted this txn (re-insert is a duplicate), false = deleted this txn (re-insert is allowed though committed has it). Keyed by cf_name + primary-key bytes; cleared at commit/rollback. */ - tidesdb_txn_t *dup_rtxn{nullptr}; std::unordered_map txn_key_state; + + /* Statement-atomicity bookkeeping. A "stmt" library savepoint is armed at + statement start (external_lock) inside a multi-statement transaction so a + statement error rolls back only its own effects, not the whole txn. The + library savepoint reverts the txn op array; these members revert the + plugin-side shadow state to the same statement boundary. + + stmt_lock_marker is held_locks_head at statement start; locks acquired + during the statement sit above it and are released to it on rollback. + stmt_key_state_undo journals prior txn_key_state values per mutation + (prior: -1 absent, 0 false, 1 true) so we revert in O(statement writes) + rather than snapshotting the whole (potentially huge) map. + stmt_fts_snapshot is a copy of the small fts_meta_pending vector taken at + statement start and restored wholesale on rollback. */ + tdb_lock_request_t *stmt_lock_marker{nullptr}; + std::vector> stmt_key_state_undo; + std::vector stmt_fts_snapshot; + bool stmt_fts_dirty_snapshot{false}; }; /* @@ -964,7 +978,7 @@ class ha_tidesdb : public handler /* Bulk UPDATE / DELETE hints -- let multi-row UPDATE/DELETE share the same mid-txn commit batching as bulk INSERT so long statements don't - blow past TDB_MAX_TXN_OPS or balloon txn memory. */ + balloon the memory the txn buffers before commit. */ bool start_bulk_update() override; int end_bulk_update() override; int bulk_update_row(const uchar *old_data, const uchar *new_data,