tidesdb engine correctness, statement atomicity, and planner-accuracy

.github/workflows/mariadb-test.yml

@@ -205,7 +205,6 @@ jobs:
     - name: Install TidesDB library (macOS)
       if: runner.os == 'macOS'
       run: |
-        sudo xcode-select -s /Applications/Xcode_16.4.app/Contents/Developer
         export SDKROOT=$(xcrun --show-sdk-path)
 
         rm -rf tidesdb-lib
@@ -296,7 +295,6 @@ jobs:
       if: runner.os == 'macOS'
       working-directory: mariadb-server
       run: |
-        sudo xcode-select -s /Applications/Xcode_16.4.app/Contents/Developer
 
         sudo rm -rf /Library/Developer/CommandLineTools/SDKs/MacOSX*.sdk
 

mysql-test/suite/tidesdb/r/tidesdb_encrypted_no_compress.result

@@ -0,0 +1,64 @@
+#
+# ============================================
+# Encrypted table: data CF compression forced off, index CF keeps it
+# ============================================
+#
+CREATE TABLE enc_lz4 (
+id INT PRIMARY KEY,
+u  INT,
+v  VARCHAR(200),
+KEY k_u (u)
+) ENGINE=TIDESDB `ENCRYPTED`=YES COMPRESSION='LZ4';
+# the table still reports the requested COMPRESSION option to the user
+SHOW CREATE TABLE enc_lz4;
+Table	Create Table
+enc_lz4	CREATE TABLE `enc_lz4` (
+  `id` int(11) NOT NULL,
+  `u` int(11) DEFAULT NULL,
+  `v` varchar(200) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  KEY `k_u` (`u`)
+) ENGINE=TidesDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci `ENCRYPTED`=YES `COMPRESSION`='LZ4'
+# data CF: compression_algorithm = NONE (ciphertext is incompressible)
+compression_algorithm = NONE
+# index CF: compression_algorithm = LZ4 (unencrypted keys keep compression)
+compression_algorithm = LZ4
+#
+# ============================================
+# Control: non-encrypted table keeps the selected compression on its data CF
+# ============================================
+#
+CREATE TABLE plain_lz4 (
+id INT PRIMARY KEY,
+v  VARCHAR(200)
+) ENGINE=TIDESDB COMPRESSION='LZ4';
+# data CF: compression_algorithm = LZ4 (not encrypted, so compression stays)
+compression_algorithm = LZ4
+#
+# ============================================
+# Data still round-trips under an encrypted, compression-disabled CF
+# ============================================
+#
+INSERT INTO enc_lz4 VALUES (1, 10, 'alpha'), (2, 20, 'beta'), (3, 30, 'gamma');
+UPDATE enc_lz4 SET v = 'delta' WHERE id = 2;
+DELETE FROM enc_lz4 WHERE id = 3;
+SELECT * FROM enc_lz4 ORDER BY id;
+id	u	v
+1	10	alpha
+2	20	delta
+# covering index scan on the still-compressed index CF
+SELECT id FROM enc_lz4 WHERE u = 10;
+id
+1
+#
+# ============================================
+# TRUNCATE recreates the data CF with compression still off
+# ============================================
+#
+TRUNCATE TABLE enc_lz4;
+compression_algorithm = NONE
+compression_algorithm = LZ4
+DROP TABLE enc_lz4;
+DROP TABLE plain_lz4;
+#
+# Done.

mysql-test/suite/tidesdb/r/tidesdb_index_stats.result

@@ -77,7 +77,7 @@ test.t_stats	analyze	status	OK
 # After ANALYZE, the optimizer should estimate ~100 rows for k=0
 EXPLAIN SELECT * FROM t_stats WHERE k = 0;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t_stats	ref	k_idx	k_idx	4	const	2	
+1	SIMPLE	t_stats	ALL	k_idx	NULL	NULL	NULL	200	Using where
 DROP TABLE t_stats;
 #
 # ============================================

mysql-test/suite/tidesdb/r/tidesdb_info_schema.result

@@ -7,20 +7,20 @@ INSERT INTO t_info_schema VALUES (1, REPEAT('a', 100));
 INSERT INTO t_info_schema VALUES (2, REPEAT('b', 100));
 INSERT INTO t_info_schema VALUES (3, REPEAT('c', 100));
 # =+=+= data_length must be non-zero =+=+=
-FAIL: DATA_LENGTH is 0
+OK: DATA_LENGTH > 0
 # =+=+= table_rows must reflect inserted rows =+=+=
-FAIL: TABLE_ROWS < 3
+OK: TABLE_ROWS >= 3
 # =+=+= add secondary index and check index_length =+=+=
 ALTER TABLE t_info_schema ADD INDEX idx_val (val);
 SELECT COUNT(*) FROM t_info_schema;
 COUNT(*)
 3
-FAIL: INDEX_LENGTH is 0
+OK: INDEX_LENGTH > 0
 # =+=+= verify after bulk insert =+=+=
 SELECT COUNT(*) FROM t_info_schema;
 COUNT(*)
 200
-FAIL: DATA_LENGTH is 0 after bulk insert
+OK: DATA_LENGTH > 0 after bulk insert
 # =+=+= create_time must be non-null =+=+=
 OK: CREATE_TIME is set
 # =+=+= update_time must be non-null after DML =+=+=

mysql-test/suite/tidesdb/r/tidesdb_keyread_decodable.result

@@ -0,0 +1,111 @@
+#
+# ============================================
+# Decodable column types: covering (Using index) IS offered
+# ============================================
+#
+CREATE TABLE cov_ok (
+id INT PRIMARY KEY,
+i  INT,
+b  BIGINT,
+d  DATE,
+ts DATETIME,
+c  CHAR(8) CHARACTER SET binary,
+KEY k_i  (i),
+KEY k_b  (b),
+KEY k_d  (d),
+KEY k_ts (ts),
+KEY k_c  (c)
+) ENGINE=TIDESDB;
+INSERT INTO cov_ok VALUES
+(1, 10, 100, '2020-01-01', '2020-01-01 10:00:00', 'aaa'),
+(2, 20, 200, '2021-02-02', '2021-02-02 11:00:00', 'bbb'),
+(3, 30, 300, '2022-03-03', '2022-03-03 12:00:00', 'ccc');
+EXPLAIN SELECT i  FROM cov_ok FORCE INDEX (k_i)  WHERE i  = 20;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cov_ok	ref	k_i	k_i	5	const	#	Using index
+EXPLAIN SELECT b  FROM cov_ok FORCE INDEX (k_b)  WHERE b  = 200;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cov_ok	ref	k_b	k_b	9	const	#	Using index
+EXPLAIN SELECT d  FROM cov_ok FORCE INDEX (k_d)  WHERE d  = '2021-02-02';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cov_ok	ref	k_d	k_d	4	const	#	Using index
+EXPLAIN SELECT ts FROM cov_ok FORCE INDEX (k_ts) WHERE ts = '2021-02-02 11:00:00';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cov_ok	ref	k_ts	k_ts	6	const	#	Using index
+EXPLAIN SELECT c  FROM cov_ok FORCE INDEX (k_c)  WHERE c  = 'bbb';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cov_ok	ref	k_c	k_c	9	const	#	Using where; Using index
+#
+# ============================================
+# Undecodable column types: covering is NOT offered (no Using index)
+# ============================================
+#
+CREATE TABLE cov_no (
+id INT PRIMARY KEY,
+s  VARCHAR(50),
+u  CHAR(8) CHARACTER SET utf8mb4,
+m  DECIMAL(10,2),
+f  DOUBLE,
+KEY k_s (s),
+KEY k_u (u),
+KEY k_m (m),
+KEY k_f (f)
+) ENGINE=TIDESDB;
+INSERT INTO cov_no VALUES
+(1, 'alpha', 'x1', 1.50, 1.5),
+(2, 'beta',  'x2', 2.50, 2.5),
+(3, 'gamma', 'x3', 3.50, 3.5);
+EXPLAIN SELECT s FROM cov_no FORCE INDEX (k_s) WHERE s = 'beta';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cov_no	ref	k_s	k_s	203	const	#	Using where
+EXPLAIN SELECT u FROM cov_no FORCE INDEX (k_u) WHERE u = 'x2';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cov_no	ref	k_u	k_u	33	const	#	Using where
+EXPLAIN SELECT m FROM cov_no FORCE INDEX (k_m) WHERE m = 2.50;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cov_no	ref	k_m	k_m	6	const	#	
+EXPLAIN SELECT f FROM cov_no FORCE INDEX (k_f) WHERE f = 2.5;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cov_no	ref	k_f	k_f	9	const	#	
+# results are still correct on the non-covering (PK fetch) path
+SELECT s FROM cov_no FORCE INDEX (k_s) WHERE s = 'beta';
+s
+beta
+SELECT m FROM cov_no FORCE INDEX (k_m) WHERE m = 2.50;
+m
+2.50
+#
+# ============================================
+# Composite index: covering decided per key part, not per whole index
+# ============================================
+#
+CREATE TABLE cov_mix (
+id INT PRIMARY KEY,
+a  INT,
+b  INT,
+s  VARCHAR(50),
+KEY k_ab (a, b),
+KEY k_as (a, s)
+) ENGINE=TIDESDB;
+INSERT INTO cov_mix VALUES (1, 1, 10, 'x'), (2, 2, 20, 'y'), (3, 3, 30, 'z');
+# (a INT, b INT): both decodable -> Using index
+EXPLAIN SELECT a, b FROM cov_mix FORCE INDEX (k_ab) WHERE a = 2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cov_mix	ref	k_ab	k_ab	5	const	#	Using index
+# (a INT, s VARCHAR), reading only the decodable a -> still Using index
+EXPLAIN SELECT a FROM cov_mix FORCE INDEX (k_as) WHERE a = 2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cov_mix	ref	k_as	k_as	5	const	#	Using index
+# (a INT, s VARCHAR), reading the undecodable s -> no Using index
+EXPLAIN SELECT a, s FROM cov_mix FORCE INDEX (k_as) WHERE a = 2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cov_mix	ref	k_as	k_as	5	const	#	
+# result on the s (non-covering) path is still correct
+SELECT a, s FROM cov_mix FORCE INDEX (k_as) WHERE a = 2;
+a	s
+2	y
+DROP TABLE cov_ok;
+DROP TABLE cov_no;
+DROP TABLE cov_mix;
+#
+# Done.

mysql-test/suite/tidesdb/r/tidesdb_mrr.result

@@ -9,7 +9,7 @@ INSERT INTO t_pk VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e'),
 # Confirm the optimizer actually picks Rowid-ordered scan (MRR).
 EXPLAIN SELECT * FROM t_pk WHERE id IN (7, 2, 9, 3, 5);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t_pk	range	PRIMARY	#	4	NULL	2	Using where
+1	SIMPLE	t_pk	range	PRIMARY	#	4	NULL	5	Using where
 # Unsorted IN-list; MRR must still return the right rows.
 SELECT * FROM t_pk WHERE id IN (7, 2, 9, 3, 5) ORDER BY id;
 id	v

mysql-test/suite/tidesdb/r/tidesdb_online_ddl.result

@@ -36,8 +36,8 @@ t_ddl	CREATE TABLE `t_ddl` (
 ALTER TABLE t_ddl ADD INDEX idx_a (a), ALGORITHM=INPLACE;
 SHOW INDEX FROM t_ddl;
 Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
-t_ddl	0	PRIMARY	1	id	A	2	NULL	NULL		LSM			NO
-t_ddl	1	idx_a	1	a	A	2	NULL	NULL	YES	LSM			NO
+t_ddl	0	PRIMARY	1	id	A	6	NULL	NULL		LSM			NO
+t_ddl	1	idx_a	1	a	A	6	NULL	NULL	YES	LSM			NO
 # Verify index is usable
 SELECT id, a FROM t_ddl WHERE a = 10 ORDER BY id;
 id	a
@@ -52,9 +52,9 @@ id	a
 ALTER TABLE t_ddl ADD INDEX idx_c (c), ALGORITHM=INPLACE;
 SHOW INDEX FROM t_ddl;
 Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
-t_ddl	0	PRIMARY	1	id	A	2	NULL	NULL		LSM			NO
-t_ddl	1	idx_a	1	a	A	2	NULL	NULL	YES	LSM			NO
-t_ddl	1	idx_c	1	c	A	2	NULL	NULL	YES	LSM			NO
+t_ddl	0	PRIMARY	1	id	A	6	NULL	NULL		LSM			NO
+t_ddl	1	idx_a	1	a	A	6	NULL	NULL	YES	LSM			NO
+t_ddl	1	idx_c	1	c	A	6	NULL	NULL	YES	LSM			NO
 EXPLAIN SELECT id, c FROM t_ddl WHERE c = 200;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t_ddl	ref	idx_c	idx_c	5	const	1	Using index
@@ -65,8 +65,8 @@ id	c
 ALTER TABLE t_ddl DROP INDEX idx_a, ALGORITHM=INPLACE;
 SHOW INDEX FROM t_ddl;
 Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
-t_ddl	0	PRIMARY	1	id	A	2	NULL	NULL		LSM			NO
-t_ddl	1	idx_c	1	c	A	2	NULL	NULL	YES	LSM			NO
+t_ddl	0	PRIMARY	1	id	A	6	NULL	NULL		LSM			NO
+t_ddl	1	idx_c	1	c	A	6	NULL	NULL	YES	LSM			NO
 # Verify remaining index still works
 SELECT id, c FROM t_ddl WHERE c = 300;
 id	c
@@ -75,8 +75,8 @@ id	c
 ALTER TABLE t_ddl ADD INDEX idx_a2 (a), DROP INDEX idx_c, ALGORITHM=INPLACE;
 SHOW INDEX FROM t_ddl;
 Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
-t_ddl	0	PRIMARY	1	id	A	2	NULL	NULL		LSM			NO
-t_ddl	1	idx_a2	1	a	A	2	NULL	NULL	YES	LSM			NO
+t_ddl	0	PRIMARY	1	id	A	6	NULL	NULL		LSM			NO
+t_ddl	1	idx_a2	1	a	A	6	NULL	NULL	YES	LSM			NO
 EXPLAIN SELECT id, a FROM t_ddl WHERE a = 20;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t_ddl	ref	idx_a2	idx_a2	5	const	1	Using index

mysql-test/suite/tidesdb/r/tidesdb_per_index_btree.result

@@ -12,9 +12,9 @@ INSERT INTO t1 VALUES (1,10,100),(2,20,200),(3,30,300);
 # idx_a should show BTREE, idx_b should show LSM
 SHOW KEYS FROM t1;
 Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
-t1	0	PRIMARY	1	id	A	2	NULL	NULL		LSM			NO
-t1	1	idx_a	1	a	A	2	NULL	NULL	YES	BTREE			NO
-t1	1	idx_b	1	b	A	2	NULL	NULL	YES	LSM			NO
+t1	0	PRIMARY	1	id	A	3	NULL	NULL		LSM			NO
+t1	1	idx_a	1	a	A	3	NULL	NULL	YES	BTREE			NO
+t1	1	idx_b	1	b	A	3	NULL	NULL	YES	LSM			NO
 SELECT * FROM t1 WHERE a = 20;
 id	a	b
 2	20	200

mysql-test/suite/tidesdb/r/tidesdb_row_cardinality.result

@@ -0,0 +1,51 @@
+CREATE TABLE t_a (id INT PRIMARY KEY, v INT, KEY k_v (v)) ENGINE=TIDESDB;
+CREATE TABLE t_b (id INT PRIMARY KEY, v INT) ENGINE=TIDESDB;
+ANALYZE TABLE t_a;
+Table	Op	Msg_type	Msg_text
+test.t_a	analyze	status	Engine-independent statistics collected
+test.t_a	analyze	Note	[TIDESDB] CF 'test__t_a'  total_keys=200  data_size=0 bytes  memtable=4600 bytes  levels=1  read_amp=1.00  cache_hit=0.0%
+test.t_a	analyze	Note	[TIDESDB] avg_key=6.9 bytes  avg_value=16.1 bytes
+test.t_a	analyze	Note	[TIDESDB] level 1  sstables=0  size=0 bytes  keys=0
+test.t_a	analyze	Note	[TIDESDB] WA  user=3800  wal=0  flush=0 (0 ssts)  compact_write=0 (0 ssts)  compact_read=0  ratio=0.00x
+test.t_a	analyze	Note	[TIDESDB] idx CF 'test__t_a__idx_k_v'  keys=200  data_size=0 bytes  levels=1
+test.t_a	analyze	Note	[TIDESDB] idx 'k_v' sampled=200 distinct=10 rec_per_key=20
+test.t_a	analyze	status	OK
+ANALYZE TABLE t_b;
+Table	Op	Msg_type	Msg_text
+test.t_b	analyze	status	Engine-independent statistics collected
+test.t_b	analyze	Note	[TIDESDB] CF 'test__t_b'  total_keys=75  data_size=0 bytes  memtable=1725 bytes  levels=1  read_amp=1.00  cache_hit=0.0%
+test.t_b	analyze	Note	[TIDESDB] avg_key=6.9 bytes  avg_value=16.1 bytes
+test.t_b	analyze	Note	[TIDESDB] level 1  sstables=0  size=0 bytes  keys=0
+test.t_b	analyze	Note	[TIDESDB] WA  user=1425  wal=0  flush=0 (0 ssts)  compact_write=0 (0 ssts)  compact_read=0  ratio=0.00x
+test.t_b	analyze	status	OK
+# each table reports its own distinct row count, not the shared-memtable sum
+# t_a -> 200
+SELECT TABLE_ROWS FROM information_schema.TABLES
+WHERE table_schema = DATABASE() AND table_name = 't_a';
+TABLE_ROWS
+200
+# t_b -> 75
+SELECT TABLE_ROWS FROM information_schema.TABLES
+WHERE table_schema = DATABASE() AND table_name = 't_b';
+TABLE_ROWS
+75
+# inserting more rows raises the estimate on the next refresh
+ANALYZE TABLE t_a;
+Table	Op	Msg_type	Msg_text
+test.t_a	analyze	status	Engine-independent statistics collected
+test.t_a	analyze	Note	[TIDESDB] CF 'test__t_a'  total_keys=300  data_size=0 bytes  memtable=6900 bytes  levels=1  read_amp=1.00  cache_hit=0.0%
+test.t_a	analyze	Note	[TIDESDB] avg_key=6.9 bytes  avg_value=16.1 bytes
+test.t_a	analyze	Note	[TIDESDB] level 1  sstables=0  size=0 bytes  keys=0
+test.t_a	analyze	Note	[TIDESDB] WA  user=5700  wal=0  flush=0 (0 ssts)  compact_write=0 (0 ssts)  compact_read=0  ratio=0.00x
+test.t_a	analyze	Note	[TIDESDB] idx CF 'test__t_a__idx_k_v'  keys=300  data_size=0 bytes  levels=1
+test.t_a	analyze	Note	[TIDESDB] idx 'k_v' sampled=300 distinct=10 rec_per_key=30
+test.t_a	analyze	status	OK
+# t_a -> 300
+SELECT TABLE_ROWS FROM information_schema.TABLES
+WHERE table_schema = DATABASE() AND table_name = 't_a';
+TABLE_ROWS
+300
+DROP TABLE t_a;
+DROP TABLE t_b;
+#
+# Done.