perf: consolidate the Frames indexes, and drop unused and duplicate indexes elsewhere

db/zm_create.sql.in

@@ -193,7 +193,6 @@ CREATE TABLE `EncoderTemplates` (
   `Description` text,
   `Params` text NOT NULL,
   PRIMARY KEY (`Id`),
-  KEY `Encoder` (`Encoder`),
   UNIQUE KEY `Encoder_Name` (`Encoder`, `Name`)
 ) ENGINE=@ZM_MYSQL_ENGINE@;
 
@@ -449,9 +448,10 @@ CREATE TABLE `Frames` (
   `Delta` decimal(8,2) NOT NULL default '0.00',
   `Score` smallint(5) unsigned NOT NULL default '0',
   PRIMARY KEY (`Id`),
-  INDEX `EventId_idx` (`EventId`),
-  KEY `Type` (`Type`),
-  KEY `TimeStamp` (`TimeStamp`)
+  /* Every query on Frames is anchored on EventId and orders or ranges by
+     FrameId within the event, so one composite index serves them all. Nothing
+     reads Frames by Type or TimeStamp without EventId. See zm_update-1.39.27. */
+  INDEX `EventId_FrameId_idx` (`EventId`,`FrameId`)
 ) ENGINE=@ZM_MYSQL_ENGINE@;
 
 --
@@ -528,7 +528,6 @@ CREATE TABLE `Role_Groups_Permissions` (
 ) ENGINE=@ZM_MYSQL_ENGINE@;
 
 CREATE UNIQUE INDEX `Role_Groups_Permissions_RoleId_GroupId_idx` ON `Role_Groups_Permissions` (`RoleId`,`GroupId`);
-CREATE INDEX `Role_Groups_Permissions_RoleId_idx` ON `Role_Groups_Permissions` (`RoleId`);
 
 --
 -- Table structure for table `Role_Monitors_Permissions`
@@ -546,7 +545,6 @@ CREATE TABLE `Role_Monitors_Permissions` (
 ) ENGINE=@ZM_MYSQL_ENGINE@;
 
 CREATE UNIQUE INDEX `Role_Monitors_Permissions_RoleId_MonitorId_idx` ON `Role_Monitors_Permissions` (`RoleId`,`MonitorId`);
-CREATE INDEX `Role_Monitors_Permissions_RoleId_idx` ON `Role_Monitors_Permissions` (`RoleId`);
 
 --
 -- Table structure for table `Logs`
@@ -564,8 +562,7 @@ CREATE TABLE `Logs` (
   `Message` text NOT NULL,
   `File` varchar(255) DEFAULT NULL,
   `Line` smallint(5) unsigned DEFAULT NULL,
-  PRIMARY KEY (`Id`),
-  KEY `TimeKey` (`TimeKey`)
+  PRIMARY KEY (`Id`)
 ) ENGINE=@ZM_MYSQL_ENGINE@;
 
 CREATE INDEX `Logs_TimeKey_idx` ON `Logs` (`TimeKey`);
@@ -791,7 +788,6 @@ CREATE TABLE `Monitor_Status` (
   `UpdatedOn`     TIMESTAMP NOT NULL default CURRENT_TIMESTAMP,
   PRIMARY KEY (`MonitorId`)
 ) ENGINE=@ZM_MYSQL_ENGINE@;
-CREATE INDEX Monitor_Status_UpdatedOn_idx on Monitor_Status(UpdatedOn);
 
 DROP TABLE IF EXISTS `Event_Summaries`;
 CREATE TABLE `Event_Summaries` (
@@ -913,9 +909,7 @@ CREATE TABLE `Stats` (
   `MaxY` smallint(5) unsigned NOT NULL default '0',
   `Score` smallint(5) unsigned NOT NULL default '0',
   PRIMARY KEY (`Id`),
-  KEY `EventId_ZoneId` (`EventId`, `ZoneId`),
-  KEY `MonitorId` (`MonitorId`),
-  KEY `ZoneId` (`ZoneId`)
+  KEY `EventId_ZoneId` (`EventId`, `ZoneId`)
 ) ENGINE=@ZM_MYSQL_ENGINE@;
 
 --

db/zm_update-1.39.27.sql

@@ -0,0 +1,54 @@
+--
+-- This updates a 1.39.26 database to 1.39.27
+--
+-- Replace the three secondary indexes on Frames with one on (EventId, FrameId).
+--
+-- Every query against Frames is anchored on EventId, and nearly all of them
+-- order or range by FrameId within the event:
+--
+--   zm_eventstream.cpp, zma.cpp    WHERE EventId=? ORDER BY FrameId ASC
+--   includes/Event.php, status.php WHERE EventId=? AND FrameId </> ? ORDER BY FrameId LIMIT 1
+--   zmfilter.pl                    WHERE EventId=? AND Type=? ORDER BY FrameId
+--   Event.pm, zmaudit.pl           max(FrameId), max(TimeStamp), sum(Score) WHERE EventId=?
+--
+-- The plain EventId index left every one of those to a filesort, which for the
+-- LIMIT 1 prev/next-frame lookups on the playback path meant reading and
+-- sorting an entire event's frames to return one row. The composite index
+-- serves the ordering directly and answers max(FrameId) as an index lookup.
+--
+-- Nothing filters or sorts on Type or TimeStamp without EventId, so neither
+-- index was ever used for reading. Type is a three-value enum, which cannot be
+-- selective in any case. Both only cost insert time on the highest-insert-rate
+-- table in the schema, plus space, plus work on every DELETE ... WHERE EventId.
+--
+-- Ordering matters here: the composite index is added first so that its
+-- leftmost prefix covers EventId for any install still carrying a foreign key
+-- on Frames.EventId (added in 1.35.11, dropped in 1.37.31 only when it was
+-- named Frames_ibfk_1). Dropping EventId_idx first would fail on those.
+--
+-- Note for large installs: the ADD INDEX is an InnoDB in-place build and does
+-- not block writes, but on a Frames table with hundreds of millions of rows it
+-- will take a while. The three drops are effectively instant.
+--
+
+SELECT 'Adding EventId_FrameId_idx to Frames. On a large Frames table this will take some time.';
+
+set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Frames' and index_name = 'EventId_FrameId_idx');
+set @sqlstmt := if( @exist = 0, 'ALTER TABLE `Frames` ADD INDEX `EventId_FrameId_idx` (`EventId`,`FrameId`)', "SELECT 'EventId_FrameId_idx INDEX already exists on Frames.'");
+PREPARE stmt FROM @sqlstmt;
+EXECUTE stmt;
+
+set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Frames' and index_name = 'EventId_idx');
+set @sqlstmt := if( @exist > 0, 'DROP INDEX `EventId_idx` ON `Frames`', "SELECT 'EventId_idx INDEX is already removed from Frames.'");
+PREPARE stmt FROM @sqlstmt;
+EXECUTE stmt;
+
+set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Frames' and index_name = 'Type');
+set @sqlstmt := if( @exist > 0, 'DROP INDEX `Type` ON `Frames`', "SELECT 'Type INDEX is already removed from Frames.'");
+PREPARE stmt FROM @sqlstmt;
+EXECUTE stmt;
+
+set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Frames' and index_name = 'TimeStamp');
+set @sqlstmt := if( @exist > 0, 'DROP INDEX `TimeStamp` ON `Frames`', "SELECT 'TimeStamp INDEX is already removed from Frames.'");
+PREPARE stmt FROM @sqlstmt;
+EXECUTE stmt;

db/zm_update-1.39.28.sql

@@ -0,0 +1,69 @@
+--
+-- This updates a 1.39.27 database to 1.39.28
+--
+-- Drop secondary indexes that no query uses, or that duplicate another index.
+--
+-- Logs.TimeKey duplicates Logs_TimeKey_idx, created immediately below it in
+-- zm_create.sql.in since 1.31.11. Both exist on every install, fresh and
+-- upgraded, and every log INSERT has maintained both ever since.
+--
+-- Stats.MonitorId and Stats.ZoneId are never read. Every query against Stats is
+-- anchored on EventId - the per-frame zone stats view, the ZoneId filter term
+-- in FilterTerm.php and Filter.pm, the deletes in Event.php and Event.pm - and
+-- EventId_ZoneId serves all of them. A Stats row is written per frame per zone
+-- when ZM_RECORD_EVENT_STATS is on, so this is the same insert-rate argument as
+-- the Frames indexes in 1.39.27.
+--
+-- EncoderTemplates.Encoder is the leftmost prefix of Encoder_Name, and the
+-- RoleId indexes on the two role permission tables are the leftmost prefix of
+-- the UNIQUE index beside them. Those tables are small and rarely written, so
+-- this is tidying rather than a saving, folded in while we are here.
+--
+-- Monitor_Status_UpdatedOn_idx is dropped by zm_update-1.37.76.sql but was left
+-- in zm_create.sql.in, so fresh installs have had it and upgraded installs have
+-- not. This drop is repeated here only so that the two agree from 1.39.28 on;
+-- installs that already ran 1.37.76 will report it as already removed.
+--
+--
+-- Stats.MonitorId and Stats.ZoneId are the only two here with no other index
+-- covering their column, so on an install still carrying a foreign key on those
+-- columns - 1.37.31 drops them only when they are named Stats_ibfk_1..4 - the
+-- drop would fail and abort the upgrade. Those two are guarded to skip instead.
+--
+
+set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Logs' and index_name = 'TimeKey');
+set @sqlstmt := if( @exist > 0, 'DROP INDEX `TimeKey` ON `Logs`', "SELECT 'TimeKey INDEX is already removed from Logs.'");
+PREPARE stmt FROM @sqlstmt;
+EXECUTE stmt;
+
+set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Stats' and index_name = 'MonitorId');
+set @fk := (select count(*) from information_schema.key_column_usage where table_schema = database() and table_name = 'Stats' and column_name = 'MonitorId' and referenced_table_name is not null);
+set @sqlstmt := if( @exist = 0, "SELECT 'MonitorId INDEX is already removed from Stats.'", if( @fk > 0, "SELECT 'Keeping Stats.MonitorId: a FOREIGN KEY still needs it.'", 'DROP INDEX `MonitorId` ON `Stats`' ));
+PREPARE stmt FROM @sqlstmt;
+EXECUTE stmt;
+
+set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Stats' and index_name = 'ZoneId');
+set @fk := (select count(*) from information_schema.key_column_usage where table_schema = database() and table_name = 'Stats' and column_name = 'ZoneId' and referenced_table_name is not null);
+set @sqlstmt := if( @exist = 0, "SELECT 'ZoneId INDEX is already removed from Stats.'", if( @fk > 0, "SELECT 'Keeping Stats.ZoneId: a FOREIGN KEY still needs it.'", 'DROP INDEX `ZoneId` ON `Stats`' ));
+PREPARE stmt FROM @sqlstmt;
+EXECUTE stmt;
+
+set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'EncoderTemplates' and index_name = 'Encoder');
+set @sqlstmt := if( @exist > 0, 'DROP INDEX `Encoder` ON `EncoderTemplates`', "SELECT 'Encoder INDEX is already removed from EncoderTemplates.'");
+PREPARE stmt FROM @sqlstmt;
+EXECUTE stmt;
+
+set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Role_Groups_Permissions' and index_name = 'Role_Groups_Permissions_RoleId_idx');
+set @sqlstmt := if( @exist > 0, 'DROP INDEX `Role_Groups_Permissions_RoleId_idx` ON `Role_Groups_Permissions`', "SELECT 'Role_Groups_Permissions_RoleId_idx INDEX is already removed from Role_Groups_Permissions.'");
+PREPARE stmt FROM @sqlstmt;
+EXECUTE stmt;
+
+set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Role_Monitors_Permissions' and index_name = 'Role_Monitors_Permissions_RoleId_idx');
+set @sqlstmt := if( @exist > 0, 'DROP INDEX `Role_Monitors_Permissions_RoleId_idx` ON `Role_Monitors_Permissions`', "SELECT 'Role_Monitors_Permissions_RoleId_idx INDEX is already removed from Role_Monitors_Permissions.'");
+PREPARE stmt FROM @sqlstmt;
+EXECUTE stmt;
+
+set @exist := (select count(*) from information_schema.statistics where table_schema = database() and table_name = 'Monitor_Status' and index_name = 'Monitor_Status_UpdatedOn_idx');
+set @sqlstmt := if( @exist > 0, 'DROP INDEX `Monitor_Status_UpdatedOn_idx` ON `Monitor_Status`', "SELECT 'Monitor_Status_UpdatedOn_idx INDEX is already removed from Monitor_Status.'");
+PREPARE stmt FROM @sqlstmt;
+EXECUTE stmt;

version.txt

@@ -1 +1 @@
-1.39.26
+1.39.28