From 66881d8a76c0545af37eed77e39aafdf424b3262 Mon Sep 17 00:00:00 2001 From: Isaac Connor Date: Mon, 7 Sep 2026 11:19:12 -0400 Subject: [PATCH 1/2] perf: replace the three Frames secondary indexes with one on (EventId, FrameId) Every query against Frames is anchored on EventId, and nearly all of them order or range by FrameId within the event: the full-event reads in zm_eventstream.cpp and zma.cpp, the prev/next bulk frame LIMIT 1 lookups in includes/Event.php and ajax/status.php, zmfilter.pl's per-type scan, and the per-event aggregates in Event.pm and zmaudit.pl. A plain EventId index left all of those to a filesort. For the LIMIT 1 lookups on the playback path that meant reading and sorting an entire event's frames to return one row. Measured on a 200 event x 500 frame table, the prev-frame query goes from "ref EventId_idx, rows 500, Using filesort" to "range EventId_FrameId_idx, Using index condition, Backward index scan", the full-event ORDER BY FrameId loses its filesort, and max(FrameId) WHERE EventId becomes "Select tables optimized away". Nothing filters or sorts on Type or TimeStamp without EventId, so neither index was ever used for reading. Type is a three-value enum and cannot be selective in any case. Both only cost insert time on the highest-insert-rate table in the schema, plus space, plus work on every DELETE ... WHERE EventId. The migration adds the composite index before dropping EventId_idx so that its leftmost prefix covers EventId for any install still carrying the foreign key on Frames.EventId that 1.35.11 added and 1.37.31 drops only when it is named Frames_ibfk_1. Verified against a schema with that foreign key present: the drop succeeds and the constraint survives. Verified idempotent by running it twice. The index is left non-unique. FrameId is unique within an event, but a UNIQUE constraint would turn a duplicate-id bug into frames dropped mid-recording rather than a log line. Co-Authored-By: Claude Opus 5 --- db/zm_create.sql.in | 7 +++--- db/zm_update-1.39.27.sql | 54 ++++++++++++++++++++++++++++++++++++++++ version.txt | 2 +- 3 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 db/zm_update-1.39.27.sql diff --git a/db/zm_create.sql.in b/db/zm_create.sql.in index 541c0b1f0f..bda0b47f74 100644 --- a/db/zm_create.sql.in +++ b/db/zm_create.sql.in @@ -449,9 +449,10 @@ CREATE TABLE `Frames` ( `Delta` decimal(8,2) NOT NULL default '0.00', `Score` smallint(5) unsigned NOT NULL default '0', PRIMARY KEY (`Id`), - INDEX `EventId_idx` (`EventId`), - KEY `Type` (`Type`), - KEY `TimeStamp` (`TimeStamp`) + /* Every query on Frames is anchored on EventId and orders or ranges by + FrameId within the event, so one composite index serves them all. Nothing + reads Frames by Type or TimeStamp without EventId. See zm_update-1.39.27. */ + INDEX `EventId_FrameId_idx` (`EventId`,`FrameId`) ) ENGINE=@ZM_MYSQL_ENGINE@; -- diff --git a/db/zm_update-1.39.27.sql b/db/zm_update-1.39.27.sql new file mode 100644 index 0000000000..fc179e7405 --- /dev/null +++ b/db/zm_update-1.39.27.sql @@ -0,0 +1,54 @@ +-- +-- This updates a 1.39.26 database to 1.39.27 +-- +-- Replace the three secondary indexes on Frames with one on (EventId, FrameId). +-- +-- Every query against Frames is anchored on EventId, and nearly all of them +-- order or range by FrameId within the event: +-- +-- zm_eventstream.cpp, zma.cpp WHERE EventId=? ORDER BY FrameId ASC +-- includes/Event.php, status.php WHERE EventId=? AND FrameId ? ORDER BY FrameId LIMIT 1 +-- zmfilter.pl WHERE EventId=? AND Type=? ORDER BY FrameId +-- Event.pm, zmaudit.pl max(FrameId), max(TimeStamp), sum(Score) WHERE EventId=? +-- +-- The plain EventId index left every one of those to a filesort, which for the +-- LIMIT 1 prev/next-frame lookups on the playback path meant reading and +-- sorting an entire event's frames to return one row. The composite index +-- serves the ordering directly and answers max(FrameId) as an index lookup. +-- +-- Nothing filters or sorts on Type or TimeStamp without EventId, so neither +-- index was ever used for reading. Type is a three-value enum, which cannot be +-- selective in any case. Both only cost insert time on the highest-insert-rate +-- table in the schema, plus space, plus work on every DELETE ... WHERE EventId. +-- +-- Ordering matters here: the composite index is added first so that its +-- leftmost prefix covers EventId for any install still carrying a foreign key +-- on Frames.EventId (added in 1.35.11, dropped in 1.37.31 only when it was +-- named Frames_ibfk_1). Dropping EventId_idx first would fail on those. +-- +-- Note for large installs: the ADD INDEX is an InnoDB in-place build and does +-- not block writes, but on a Frames table with hundreds of millions of rows it +-- will take a while. The three drops are effectively instant. +-- + +SELECT 'Adding EventId_FrameId_idx to Frames. On a large Frames table this will take some time.'; + +set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Frames' and index_name = 'EventId_FrameId_idx'); +set @sqlstmt := if( @exist = 0, 'ALTER TABLE `Frames` ADD INDEX `EventId_FrameId_idx` (`EventId`,`FrameId`)', "SELECT 'EventId_FrameId_idx INDEX already exists on Frames.'"); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Frames' and index_name = 'EventId_idx'); +set @sqlstmt := if( @exist > 0, 'DROP INDEX `EventId_idx` ON `Frames`', "SELECT 'EventId_idx INDEX is already removed from Frames.'"); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Frames' and index_name = 'Type'); +set @sqlstmt := if( @exist > 0, 'DROP INDEX `Type` ON `Frames`', "SELECT 'Type INDEX is already removed from Frames.'"); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Frames' and index_name = 'TimeStamp'); +set @sqlstmt := if( @exist > 0, 'DROP INDEX `TimeStamp` ON `Frames`', "SELECT 'TimeStamp INDEX is already removed from Frames.'"); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; diff --git a/version.txt b/version.txt index 14fdf1dcda..6ec64f311e 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.39.26 +1.39.27 From 11bade086e3ddd45c9199125c0031c433f089a34 Mon Sep 17 00:00:00 2001 From: Isaac Connor Date: Mon, 7 Sep 2026 11:51:49 -0400 Subject: [PATCH 2/2] perf: drop unused and duplicate secondary indexes An audit of every table in zm_create.sql.in against the queries that read it turned up seven indexes that are either never used or duplicate another index. Logs.TimeKey duplicates Logs_TimeKey_idx, which zm_create.sql.in creates three lines below it and zm_update-1.31.11.sql adds to upgraded installs. Both have existed on every install since, and every log INSERT maintains both. Logs is the second highest insert-rate table in the schema. Stats.MonitorId and Stats.ZoneId are never read. Every query against Stats is anchored on EventId: the per-frame zone stats view in skins functions.php and ajax/stats.php, the ZoneId filter term in FilterTerm.php and Filter.pm, the deletes in Event.php, Event.pm and zmaudit.pl, and the orphan scan. EventId_ZoneId serves all of them. A Stats row is written per frame per zone when ZM_RECORD_EVENT_STATS is on, which is the same insert-rate argument as the Frames indexes in the previous commit. EncoderTemplates.Encoder is the leftmost prefix of Encoder_Name, and the RoleId indexes on Role_Groups_Permissions and Role_Monitors_Permissions are the leftmost prefix of the UNIQUE index beside them. Those tables are small and rarely written, so this is tidying rather than a saving. Monitor_Status_UpdatedOn_idx is dropped by zm_update-1.37.76.sql but was left in zm_create.sql.in, so fresh installs have carried it and upgraded installs have not. Removed from zm_create.sql.in and repeated in the migration so the two agree from here on. Stats.MonitorId and Stats.ZoneId are the only two with no other index covering their column, so their drop is guarded on there being no foreign key on the column. 1.37.31 drops the Stats foreign keys only when they are named Stats_ibfk_1..4; on an install where they survived under another name the drop would fail with errno 150 and abort the upgrade, so it skips with a message instead. Verified by building that case and confirming the migration completes. Tested against MySQL 8.4: the schema built from master's zm_create.sql.in, with 1.39.27 and 1.39.28 applied, is index for index identical to a fresh install from the updated zm_create.sql.in - 123 indexes across 55 tables. Both migrations verified idempotent, and all 29 foreign keys survive. Not changed: AI_Detections carries three single-column indexes and has no consumer anywhere in the tree yet, so there is nothing to judge them against. Co-Authored-By: Claude Opus 5 --- db/zm_create.sql.in | 11 ++----- db/zm_update-1.39.28.sql | 69 ++++++++++++++++++++++++++++++++++++++++ version.txt | 2 +- 3 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 db/zm_update-1.39.28.sql diff --git a/db/zm_create.sql.in b/db/zm_create.sql.in index bda0b47f74..980d46e8a8 100644 --- a/db/zm_create.sql.in +++ b/db/zm_create.sql.in @@ -193,7 +193,6 @@ CREATE TABLE `EncoderTemplates` ( `Description` text, `Params` text NOT NULL, PRIMARY KEY (`Id`), - KEY `Encoder` (`Encoder`), UNIQUE KEY `Encoder_Name` (`Encoder`, `Name`) ) ENGINE=@ZM_MYSQL_ENGINE@; @@ -529,7 +528,6 @@ CREATE TABLE `Role_Groups_Permissions` ( ) ENGINE=@ZM_MYSQL_ENGINE@; CREATE UNIQUE INDEX `Role_Groups_Permissions_RoleId_GroupId_idx` ON `Role_Groups_Permissions` (`RoleId`,`GroupId`); -CREATE INDEX `Role_Groups_Permissions_RoleId_idx` ON `Role_Groups_Permissions` (`RoleId`); -- -- Table structure for table `Role_Monitors_Permissions` @@ -547,7 +545,6 @@ CREATE TABLE `Role_Monitors_Permissions` ( ) ENGINE=@ZM_MYSQL_ENGINE@; CREATE UNIQUE INDEX `Role_Monitors_Permissions_RoleId_MonitorId_idx` ON `Role_Monitors_Permissions` (`RoleId`,`MonitorId`); -CREATE INDEX `Role_Monitors_Permissions_RoleId_idx` ON `Role_Monitors_Permissions` (`RoleId`); -- -- Table structure for table `Logs` @@ -565,8 +562,7 @@ CREATE TABLE `Logs` ( `Message` text NOT NULL, `File` varchar(255) DEFAULT NULL, `Line` smallint(5) unsigned DEFAULT NULL, - PRIMARY KEY (`Id`), - KEY `TimeKey` (`TimeKey`) + PRIMARY KEY (`Id`) ) ENGINE=@ZM_MYSQL_ENGINE@; CREATE INDEX `Logs_TimeKey_idx` ON `Logs` (`TimeKey`); @@ -792,7 +788,6 @@ CREATE TABLE `Monitor_Status` ( `UpdatedOn` TIMESTAMP NOT NULL default CURRENT_TIMESTAMP, PRIMARY KEY (`MonitorId`) ) ENGINE=@ZM_MYSQL_ENGINE@; -CREATE INDEX Monitor_Status_UpdatedOn_idx on Monitor_Status(UpdatedOn); DROP TABLE IF EXISTS `Event_Summaries`; CREATE TABLE `Event_Summaries` ( @@ -914,9 +909,7 @@ CREATE TABLE `Stats` ( `MaxY` smallint(5) unsigned NOT NULL default '0', `Score` smallint(5) unsigned NOT NULL default '0', PRIMARY KEY (`Id`), - KEY `EventId_ZoneId` (`EventId`, `ZoneId`), - KEY `MonitorId` (`MonitorId`), - KEY `ZoneId` (`ZoneId`) + KEY `EventId_ZoneId` (`EventId`, `ZoneId`) ) ENGINE=@ZM_MYSQL_ENGINE@; -- diff --git a/db/zm_update-1.39.28.sql b/db/zm_update-1.39.28.sql new file mode 100644 index 0000000000..c5688aa8cb --- /dev/null +++ b/db/zm_update-1.39.28.sql @@ -0,0 +1,69 @@ +-- +-- This updates a 1.39.27 database to 1.39.28 +-- +-- Drop secondary indexes that no query uses, or that duplicate another index. +-- +-- Logs.TimeKey duplicates Logs_TimeKey_idx, created immediately below it in +-- zm_create.sql.in since 1.31.11. Both exist on every install, fresh and +-- upgraded, and every log INSERT has maintained both ever since. +-- +-- Stats.MonitorId and Stats.ZoneId are never read. Every query against Stats is +-- anchored on EventId - the per-frame zone stats view, the ZoneId filter term +-- in FilterTerm.php and Filter.pm, the deletes in Event.php and Event.pm - and +-- EventId_ZoneId serves all of them. A Stats row is written per frame per zone +-- when ZM_RECORD_EVENT_STATS is on, so this is the same insert-rate argument as +-- the Frames indexes in 1.39.27. +-- +-- EncoderTemplates.Encoder is the leftmost prefix of Encoder_Name, and the +-- RoleId indexes on the two role permission tables are the leftmost prefix of +-- the UNIQUE index beside them. Those tables are small and rarely written, so +-- this is tidying rather than a saving, folded in while we are here. +-- +-- Monitor_Status_UpdatedOn_idx is dropped by zm_update-1.37.76.sql but was left +-- in zm_create.sql.in, so fresh installs have had it and upgraded installs have +-- not. This drop is repeated here only so that the two agree from 1.39.28 on; +-- installs that already ran 1.37.76 will report it as already removed. +-- +-- +-- Stats.MonitorId and Stats.ZoneId are the only two here with no other index +-- covering their column, so on an install still carrying a foreign key on those +-- columns - 1.37.31 drops them only when they are named Stats_ibfk_1..4 - the +-- drop would fail and abort the upgrade. Those two are guarded to skip instead. +-- + +set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Logs' and index_name = 'TimeKey'); +set @sqlstmt := if( @exist > 0, 'DROP INDEX `TimeKey` ON `Logs`', "SELECT 'TimeKey INDEX is already removed from Logs.'"); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Stats' and index_name = 'MonitorId'); +set @fk := (select count(*) from information_schema.key_column_usage where table_schema = database() and table_name = 'Stats' and column_name = 'MonitorId' and referenced_table_name is not null); +set @sqlstmt := if( @exist = 0, "SELECT 'MonitorId INDEX is already removed from Stats.'", if( @fk > 0, "SELECT 'Keeping Stats.MonitorId: a FOREIGN KEY still needs it.'", 'DROP INDEX `MonitorId` ON `Stats`' )); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Stats' and index_name = 'ZoneId'); +set @fk := (select count(*) from information_schema.key_column_usage where table_schema = database() and table_name = 'Stats' and column_name = 'ZoneId' and referenced_table_name is not null); +set @sqlstmt := if( @exist = 0, "SELECT 'ZoneId INDEX is already removed from Stats.'", if( @fk > 0, "SELECT 'Keeping Stats.ZoneId: a FOREIGN KEY still needs it.'", 'DROP INDEX `ZoneId` ON `Stats`' )); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'EncoderTemplates' and index_name = 'Encoder'); +set @sqlstmt := if( @exist > 0, 'DROP INDEX `Encoder` ON `EncoderTemplates`', "SELECT 'Encoder INDEX is already removed from EncoderTemplates.'"); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Role_Groups_Permissions' and index_name = 'Role_Groups_Permissions_RoleId_idx'); +set @sqlstmt := if( @exist > 0, 'DROP INDEX `Role_Groups_Permissions_RoleId_idx` ON `Role_Groups_Permissions`', "SELECT 'Role_Groups_Permissions_RoleId_idx INDEX is already removed from Role_Groups_Permissions.'"); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Role_Monitors_Permissions' and index_name = 'Role_Monitors_Permissions_RoleId_idx'); +set @sqlstmt := if( @exist > 0, 'DROP INDEX `Role_Monitors_Permissions_RoleId_idx` ON `Role_Monitors_Permissions`', "SELECT 'Role_Monitors_Permissions_RoleId_idx INDEX is already removed from Role_Monitors_Permissions.'"); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Monitor_Status' and index_name = 'Monitor_Status_UpdatedOn_idx'); +set @sqlstmt := if( @exist > 0, 'DROP INDEX `Monitor_Status_UpdatedOn_idx` ON `Monitor_Status`', "SELECT 'Monitor_Status_UpdatedOn_idx INDEX is already removed from Monitor_Status.'"); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; diff --git a/version.txt b/version.txt index 6ec64f311e..a51150d6e0 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.39.27 +1.39.28