From afdd677682a4aa91e656bedc7bce5ef574e38af4 Mon Sep 17 00:00:00 2001 From: jenunes Date: Tue, 19 May 2026 20:29:31 -0300 Subject: [PATCH] PT-2516 - enhance pt-mongodb-index-check unused and duplicate analysis Multi-signal unused index scoring: - Collect $indexStats, listIndexes, collStats, and serverStatus - Score indexes into tiers: SAFE_TO_DROP, LIKELY_UNUSED, LOW_USAGE, MONITOR, KEEP - Hard guards for _id_, unique, TTL, hidden, partial, and system indexes - Warmup period detection (configurable --warmup-days) - Cross-reference duplicate prefix indexes with container usage Duplicate detection improvements: - keyToken distinguishes hashed/text/2dsphere/2d from btree directions - Handle primitive.Symbol decoding (MongoDB 8.x) - Skip _id_ as a droppable prefix candidate - Property-aware: partialFilterExpression, sparse, collation checked - Unique prefix warning when dropping would lose uniqueness - Rich sectioned text report with Reason, Action, optional index sizes Sharded cluster support: - Aggregate $indexStats across shards (sum ops, oldest since) - NormalizeIndexStat for mongos responses missing top-level name/key - DeduplicateIndexRecords safety net Connection and UX: - Ping after connect for immediate auth/network errors - Require --databases or --all-databases (or DB in URI) - Skip system databases (admin, config, local) and system.* collections - collStats IndexSizes uses dynamic map for arbitrary index names Documentation: - README.rst rewritten with new flags, sample output, sharded behavior, edge cases (warmup, read preference, background builds), PTDEBUG usage Tests: - 41 unit/regression tests covering scoring, duplicate detection, report rendering, system DB filtering, URI parsing, and sharded aggregation Co-authored-by: Cursor --- src/go/mongolib/proto/collstats.go | 70 +- src/go/pt-mongodb-index-check/README.rst | 308 +++++++- .../indexes/analysis.go | 278 ++++++++ .../indexes/analysis_test.go | 660 ++++++++++++++++++ .../indexes/collector.go | 246 +++++++ .../indexes/duplicated.go | 164 ++++- .../pt-mongodb-index-check/indexes/unused.go | 52 +- .../indexes/unused_test.go | 14 + src/go/pt-mongodb-index-check/main.go | 461 +++++++++++- src/go/pt-mongodb-index-check/main_test.go | 272 ++++++++ .../templates/analysis.go | 42 ++ .../templates/duplicated.go | 33 +- 12 files changed, 2469 insertions(+), 131 deletions(-) create mode 100644 src/go/pt-mongodb-index-check/indexes/analysis.go create mode 100644 src/go/pt-mongodb-index-check/indexes/analysis_test.go create mode 100644 src/go/pt-mongodb-index-check/indexes/collector.go create mode 100644 src/go/pt-mongodb-index-check/templates/analysis.go diff --git a/src/go/mongolib/proto/collstats.go b/src/go/mongolib/proto/collstats.go index c0f140fc0..3fe16893c 100644 --- a/src/go/mongolib/proto/collstats.go +++ b/src/go/mongolib/proto/collstats.go @@ -1,45 +1,39 @@ package proto type ShardStas struct { - Ns string `json:"ns"` - Count int64 `json:"count"` - Size int64 `json:"size"` - AvgObjSize int64 `json:"avgObjSize"` - NumExtents int64 `json:"numExtents"` - StorageSize int64 `json:"storageSize"` - LastExtentSize int64 `json:"lastExtentSize"` - PaddingFactor int64 `json:"paddingFactor"` - PaddingFactorNote string `json:"paddingFactorNote"` - UserFlags int64 `json:"userFlags"` - Capped bool `json:"capped"` - Nindexes int64 `json:"nindexes"` - IndexDetails struct{} `json:"indexDetails"` - TotalIndexSize int64 `json:"totalIndexSize"` - IndexSizes struct { - ID int64 `json:"_id_"` - IDHashed int64 `json:"_id_hashed"` - } `json:"indexSizes"` - Ok int `json:"ok"` + Ns string `json:"ns"` + Count int64 `json:"count"` + Size int64 `json:"size"` + AvgObjSize int64 `json:"avgObjSize"` + NumExtents int64 `json:"numExtents"` + StorageSize int64 `json:"storageSize"` + LastExtentSize int64 `json:"lastExtentSize"` + PaddingFactor int64 `json:"paddingFactor"` + PaddingFactorNote string `json:"paddingFactorNote"` + UserFlags int64 `json:"userFlags"` + Capped bool `json:"capped"` + Nindexes int64 `json:"nindexes"` + IndexDetails struct{} `json:"indexDetails"` + TotalIndexSize int64 `json:"totalIndexSize"` + IndexSizes map[string]int64 `json:"indexSizes"` + Ok int `json:"ok"` } type CollStats struct { - Sharded bool `json:"sharded"` - PaddingFactorNote string `json:"paddingFactorNote"` - UserFlags int64 `json:"userFlags"` - Capped bool `json:"capped"` - Ns string `json:"ns"` - Count int64 `json:"count"` - NumExtents int64 `json:"numExtents"` - Size int64 `json:"size"` - StorageSize int64 `json:"storageSize"` - TotalIndexSize int64 `json:"totalIndexSize"` - IndexSizes struct { - ID int `json:"_id_"` - IDHashed int `json:"_id_hashed"` - } `json:"indexSizes"` - AvgObjSize int64 `json:"avgObjSize"` - Nindexes int64 `json:"nindexes"` - Nchunks int64 `json:"nchunks"` - Shards map[string]ShardStas `json:"shards"` - Ok int64 `json:"ok"` + Sharded bool `json:"sharded"` + PaddingFactorNote string `json:"paddingFactorNote"` + UserFlags int64 `json:"userFlags"` + Capped bool `json:"capped"` + Ns string `json:"ns"` + Count int64 `json:"count"` + NumExtents int64 `json:"numExtents"` + Size int64 `json:"size"` + StorageSize int64 `json:"storageSize"` + TotalIndexSize int64 `json:"totalIndexSize"` + IndexSizes map[string]int64 `json:"indexSizes"` + AvgObjSize int64 `json:"avgObjSize"` + Nindexes int64 `json:"nindexes"` + Nchunks int64 `json:"nchunks"` + Shards map[string]ShardStas `json:"shards"` + Ok int64 `json:"ok"` } diff --git a/src/go/pt-mongodb-index-check/README.rst b/src/go/pt-mongodb-index-check/README.rst index 6969f8274..2253072c3 100644 --- a/src/go/pt-mongodb-index-check/README.rst +++ b/src/go/pt-mongodb-index-check/README.rst @@ -4,7 +4,8 @@ :program:`pt-mongodb-index-check` ================================= -Performs checks on MongoDB indexes. +Performs checks on MongoDB indexes: identifies duplicated (prefix) indexes and +analyzes unused indexes with a multi-signal scoring system. Checks available ================ @@ -12,6 +13,15 @@ Checks available Duplicated indexes ~~~~~~~~~~~~~~~~~~ +System databases (``admin``, ``config``, ``local``) and internal collections +whose names start with ``system.`` (for example ``system.profile``) are not +scanned for duplicate prefixes, consistent with the unused-index path. + +The sectioned **Duplicate Prefix Index Report** (banner, per-pair reason, +``dropIndex`` action, optional index sizes from ``collStats``, and a separate +block for unique-prefix warnings) is printed in text mode for ``check-duplicates`` +and ``check-all`` only; ``check-unused`` output focuses on unused-index analysis. + Check for indexes that are the prefix of other indexes. For example if we have these 2 indexes .. code-block:: javascript @@ -23,52 +33,284 @@ Check for indexes that are the prefix of other indexes. For example if we have t The index ``idx_02`` is the prefix of ``idx_01`` because it has the same keys in the same order so, ``idx_02`` can be dropped. -Unused indexes. -~~~~~~~~~~~~~~~ +The duplicate check is property-aware: two indexes with the same key prefix are +**not** flagged as duplicates if they differ in any of the following: + +- ``partialFilterExpression`` -- indexes covering different document subsets +- ``sparse`` -- sparse and non-sparse indexes have different null-handling behavior +- ``collation`` -- indexes with different collation rules serve different queries + +The ``_id_`` index is always excluded from duplicate candidates since it is a +MongoDB requirement and cannot be dropped. + +**Index type awareness:** Hashed, text, and geospatial index types (``2dsphere``, +``2d``) are treated as distinct types in the key comparison. An index on +``{_id: 1}`` and ``{_id: "hashed"}`` are never considered prefix duplicates +because they use fundamentally different index structures. + +If a **unique** index is detected as a prefix of a non-unique container index, +a warning is emitted because dropping the unique index would remove the +uniqueness constraint. + +.. code-block:: text + + WARNING: prefix index enforces unique constraint; dropping requires the container index to also be unique + +Unused indexes (enhanced analysis) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The unused index check goes beyond the simple ``$indexStats`` ``accesses.ops = 0`` +metric. It uses a multi-signal scoring system that collects data from four sources: + +1. **$indexStats** -- read access counts and the stats reset timestamp +2. **listIndexes** -- index properties (unique, sparse, partial, TTL, hidden) +3. **collStats** -- per-index sizes, collection document count, total index size +4. **serverStatus** -- global write rate for cost estimation + +Each index is evaluated through a scoring decision tree that produces one of +these recommendations: + ++-------------------+--------------------------------------------------------------+ +| Recommendation | Meaning | ++===================+==============================================================+ +| ``SAFE_TO_DROP`` | High confidence the index provides no value; includes a | +| | ``dropIndex`` command ready to copy. | ++-------------------+--------------------------------------------------------------+ +| ``LIKELY_UNUSED`` | Strong signal of no usage, but low cost to keep (small size) | ++-------------------+--------------------------------------------------------------+ +| ``LOW_USAGE`` | Index is used but at a very low rate relative to write cost | ++-------------------+--------------------------------------------------------------+ +| ``MONITOR`` | Insufficient data to decide (warmup period, empty collection)| ++-------------------+--------------------------------------------------------------+ +| ``KEEP`` | Index enforces a constraint (unique, TTL) or is hidden | ++-------------------+--------------------------------------------------------------+ + +**Hard guards** -- The following indexes are never flagged for removal: + +- ``_id_`` (MongoDB requirement) +- Unique indexes (enforce data integrity constraints) +- TTL indexes (provide automatic document expiration) +- Hidden indexes (intentionally excluded from planner by admin) +- Indexes in system databases (admin, config, local) +- Collections whose names begin with ``system.`` (internal collections such as ``system.profile``) -This check gets the ``$indexstats`` for all indexes and reports those -having ``accesses.ops`` = 0. +**Warmup period** -- After a ``mongod`` restart, ``accesses.since`` resets and +all indexes appear to have zero ops. The tool waits for the observation window +to exceed ``--warmup-days`` (default: 7) before flagging unused indexes. + +**Cross-reference with duplicates** -- When ``check-all`` is used, a prefix- +duplicate index with zero ops whose container index has active reads is +recommended as ``SAFE_TO_DROP`` with an explanation. Usage ===== Run the program as ``pt-mongodb-index-check [flags]`` +You must specify which databases to check using ``--databases`` or +``--all-databases``. If neither is provided but the connection URI contains a +database name (e.g., ``mongodb://host:port/mydb``), that database is used as +the default. + +The tool verifies connectivity (Ping) immediately after connecting and reports +a clear error if the server is unreachable or credentials are invalid. Those +errors are written to standard error; if you paste output from several runs +or from a wrapper that merges streams, a failed authentication line can appear +above a successful report from a different invocation. + +Environment +=========== + +PTDEBUG +~~~~~~~ + +The environment variable ``PTDEBUG`` enables verbose diagnostic logging on +**standard error**, consistent with other Percona Toolkit tools. Set it to any +non-empty value except ``0`` (for example ``PTDEBUG=1``). Unset or empty turns +debugging off; ``PTDEBUG=0`` is treated as off (same as Perl toolkit +``$ENV{PTDEBUG} || 0``). + +The normal report (text templates or ``--json``) is still written to **standard +output** only. To capture everything in one file: + +.. code-block:: bash + + PTDEBUG=1 pt-mongodb-index-check check-all --mongodb.uri=mongodb://127.0.0.1:27017 --all-databases --all-collections > report.txt 2>&1 + +Diagnostic lines intentionally avoid printing raw connection passwords (the +password in ``--mongodb.uri`` is redacted in debug output). + +Debug output can grow large on clusters with many databases and collections. + Available commands ~~~~~~~~~~~~~~~~~~ -================ ================================== -Command Description -================ ================================== -check-duplicated Run checks for duplicated indexes. -check-unused Run check for unused indexes. -check-all Run all checks -================ ================================== +================= ================================== +Command Description +================= ================================== +check-duplicates Run checks for duplicated indexes. +check-unused Run check for unused indexes. +check-all Run all checks. +================= ================================== Available flags ~~~~~~~~~~~~~~~ -+----------------------------+----------------------------------------+ -| Flag | Description | -+============================+========================================+ -| –all-databases | Check in all databases excluding | -| | system dbs. | -+----------------------------+----------------------------------------+ -| –databases=DATABASES,… | Comma separated list of databases to | -| | check. | -+----------------------------+----------------------------------------+ -| –all-collections | Check in all collections in the | -| | selected databases. | -+----------------------------+----------------------------------------+ -| –collections=COLLECTIONS,… | Comma separated list of collections to | -| | check. | -+----------------------------+----------------------------------------+ -| –mongodb.uri= | Connection URI | -+----------------------------+----------------------------------------+ -| –json | Show output as JSON | -+----------------------------+----------------------------------------+ -| –version | Show version information | -+----------------------------+----------------------------------------+ ++----------------------------------+------------------------------------------+ +| Flag | Description | ++==================================+==========================================+ +| --all-databases | Check in all databases excluding | +| | system dbs. | ++----------------------------------+------------------------------------------+ +| --databases=DATABASES,... | Comma separated list of databases to | +| | check. | ++----------------------------------+------------------------------------------+ +| --all-collections | Check in all collections in the | +| | selected databases. | ++----------------------------------+------------------------------------------+ +| --collections=COLLECTIONS,... | Comma separated list of collections to | +| | check. | ++----------------------------------+------------------------------------------+ +| --mongodb.uri= | Connection URI. | ++----------------------------------+------------------------------------------+ +| --json | Show output as JSON. | ++----------------------------------+------------------------------------------+ +| --warmup-days=7 | Minimum observation window (days) | +| | before flagging unused indexes. | ++----------------------------------+------------------------------------------+ +| --low-usage-threshold=1.0 | Ops/day below which an index is | +| | considered low-usage. | ++----------------------------------+------------------------------------------+ +| --large-index-size=10485760 | Index size threshold in bytes for | +| | "large" classification (default 10 MB). | ++----------------------------------+------------------------------------------+ +| --include-low-usage | Also report indexes with low but | +| | non-zero usage. | ++----------------------------------+------------------------------------------+ +| --cross-reference-duplicates | Combine unused + duplicate analysis for | +| | better recommendations (default: true). | ++----------------------------------+------------------------------------------+ +| --version | Show version information. | ++----------------------------------+------------------------------------------+ + +Examples +======== + +Check all indexes across all databases: + +.. code-block:: bash + + pt-mongodb-index-check check-all --mongodb.uri=mongodb://127.0.0.1:27017 --all-databases --all-collections + +Check a specific database: + +.. code-block:: bash + + pt-mongodb-index-check check-unused --mongodb.uri=mongodb://127.0.0.1:27017/mydb --all-collections + +Include low-usage indexes with a custom threshold: + +.. code-block:: bash + + pt-mongodb-index-check check-unused --mongodb.uri=mongodb://127.0.0.1:27017 --databases=mydb --all-collections --include-low-usage --low-usage-threshold=0.5 + +Sample output +~~~~~~~~~~~~~ + +.. code-block:: text + + # ============================================================ + # Duplicate Prefix Index Report + # ============================================================ + # Pairs found: 1 across 1 database(s), 1 collection(s) + # ---- REDUNDANT PREFIX (shorter index is candidate to drop) --- + mydb.orders + Prefix: 'idx_region' {region:1} 4.0 KB + Container: 'idx_region_date' {region:1, date:-1} 8.0 KB + Reason: 'idx_region' is a key-order prefix of 'idx_region_date'; any query served by 'idx_region' can also use 'idx_region_date'. + Action: db.orders.dropIndex("idx_region") + + # Summary: 1 redundant prefix pair(s), 0 with unique/constraint warning(s) + + # ============================================================ + # Unused Index Analysis + # ============================================================ + # Observation window: 2024-01-01T00:00:00Z to 2024-02-15T00:00:00Z (45.0 days) + # Indexes analyzed: 8 across 1 database(s), 3 collection(s) + # Server write rate: ~2400 ops/sec + + # ---- SAFE TO DROP (high confidence) -------------------------- + + mydb.orders index 'idx_old_status' {status:1, date:-1} + Ops: 0 in 45 days | Size: 128.0 MB | Score: 0.95 + Reason: Zero reads in 45 days; index is 128.0 MB and costs write amplification + Action: db.orders.dropIndex("idx_old_status") + + # ---- LIKELY UNUSED (review recommended) ---------------------- + + mydb.users index 'idx_legacy_field' {legacyCode:1} + Ops: 0 in 45 days | Size: 2.0 MB | Score: 0.80 + Reason: Zero reads in 45 days; small index (2.0 MB), low cost to keep + + # ---- MONITOR (insufficient data) ----------------------------- + + mydb.logs index 'idx_new_feature' {featureFlag:1} + Ops: 0 in 3 days | Size: 500.0 KB | Score: 0.10 + Reason: Index created/stats reset 3 days ago; re-check after 7 days + + # ---- KEEP (constraints / special) ---------------------------- + + mydb.users index 'email_unique' {email:1} [UNIQUE] + Ops: 0 in 45 days | Kept: enforces uniqueness constraint + + # Summary: 1 safe to drop (saving ~128.0 MB), 1 likely unused, + # 0 low usage, 1 monitoring, 1 kept (constraints) + +JSON output includes the full ``IndexAnalysis`` array with all fields +(``score``, ``recommendation``, ``confidence``, ``reason``, ``indexSizeBytes``, +``ageDays``, ``opsPerDay``, etc.) for programmatic consumption. + +Edge cases and caveats +====================== + +accesses.since resets on restart +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``accesses.since`` field resets every time ``mongod`` restarts. On clusters +with frequent rolling restarts (e.g., Kubernetes pod recycling), the observation +window may be very short. The tool prints the observation window in the report +header and marks indexes in the warmup period as ``MONITOR``. + +Sharded clusters +~~~~~~~~~~~~~~~~ + +On sharded clusters, ``$indexStats`` returns one entry per shard per index. The +tool automatically aggregates these entries by index name: it sums +``accesses.ops`` across all shards and uses the oldest ``accesses.since`` as the +observation window (most conservative). This produces one row per index in the +report instead of N rows for N shards. + +When a row omits the top-level ``name`` field but includes ``spec.name`` (seen on +some mongos responses), the tool normalizes the document before aggregation so +shards still group under the same index name. + +Per-shard detail is still available via ``--json`` output, which includes the +shard count for each aggregated entry. + +Read preference routing +~~~~~~~~~~~~~~~~~~~~~~~ + +Applications using ``readPreference: secondary`` route reads to secondaries. +If ``$indexStats`` is collected only from the primary, indexes used exclusively +by secondary reads appear unused. Run with a connection URI pointing to each +replica set member or use ``readPreference=secondaryPreferred`` in the URI. + +Background index builds +~~~~~~~~~~~~~~~~~~~~~~~ + +A newly built index may have ``ops == 0`` simply because it was still building +when the check ran. The warmup period (``--warmup-days``) mitigates this for +most cases. Authors ======= diff --git a/src/go/pt-mongodb-index-check/indexes/analysis.go b/src/go/pt-mongodb-index-check/indexes/analysis.go new file mode 100644 index 000000000..ed6e0f2b7 --- /dev/null +++ b/src/go/pt-mongodb-index-check/indexes/analysis.go @@ -0,0 +1,278 @@ +package indexes + +import ( + "fmt" + "math" + "time" + + "go.mongodb.org/mongo-driver/bson/primitive" +) + +const ( + RecommendSafeToDrop = "SAFE_TO_DROP" + RecommendLikelyUnused = "LIKELY_UNUSED" + RecommendMonitor = "MONITOR" + RecommendLowUsage = "LOW_USAGE" + RecommendKeepConstraint = "KEEP_CONSTRAINT" + RecommendKeepHidden = "KEEP_HIDDEN" + RecommendKeepPartial = "KEEP_PARTIAL" + + ConfidenceHigh = "high" + ConfidenceMedium = "medium" + ConfidenceLow = "low" + + DefaultWarmupDays = 7 + DefaultLowUsageThreshold = 1.0 + DefaultLargeIndexSize = 10 * 1024 * 1024 // 10 MB +) + +// AnalysisConfig holds configurable thresholds for index analysis. +type AnalysisConfig struct { + WarmupDays float64 + LowUsageThreshold float64 + LargeIndexSizeBytes int64 + IncludeLowUsage bool + CrossReferenceDuplicates bool + Now time.Time +} + +// DefaultAnalysisConfig returns the default configuration. +func DefaultAnalysisConfig() AnalysisConfig { + return AnalysisConfig{ + WarmupDays: DefaultWarmupDays, + LowUsageThreshold: DefaultLowUsageThreshold, + LargeIndexSizeBytes: DefaultLargeIndexSize, + IncludeLowUsage: false, + CrossReferenceDuplicates: true, + Now: time.Now(), + } +} + +// IndexAnalysis holds the full analysis result for a single index. +type IndexAnalysis struct { + Namespace string `json:"namespace"` + IndexName string `json:"indexName"` + IndexKey primitive.D `json:"indexKey"` + + AccessOps int64 `json:"accessOps"` + AccessSince time.Time `json:"accessSince"` + + AgeDays float64 `json:"ageDays"` + OpsPerDay float64 `json:"opsPerDay"` + IndexSizeBytes int64 `json:"indexSizeBytes"` + CollDocCount int64 `json:"collDocCount"` + CollTotalIdxSize int64 `json:"collTotalIdxSize"` + IndexSizePct float64 `json:"indexSizePct"` + + IsPartial bool `json:"isPartial"` + IsSparse bool `json:"isSparse"` + IsUnique bool `json:"isUnique"` + IsTTL bool `json:"isTTL"` + IsHidden bool `json:"isHidden"` + + WriteOpsPerSec float64 `json:"writeOpsPerSec"` + + Score float64 `json:"score"` + Recommendation string `json:"recommendation"` + Confidence string `json:"confidence"` + Reason string `json:"reason"` +} + +// IndexRecord is the merged data collected from multiple MongoDB commands +// before scoring. It is the input to ScoreIndex. +type IndexRecord struct { + Namespace string + IndexName string + IndexKey primitive.D + + AccessOps int64 + AccessSince time.Time + + IndexSizeBytes int64 + CollDocCount int64 + CollTotalIdxSize int64 + + IsPartial bool + IsSparse bool + IsUnique bool + IsTTL bool + IsHidden bool + + WriteOpsPerSec float64 + + // Set by cross-reference with duplicate check results + IsDuplicatePrefix bool + DuplicateContainerName string + DuplicateContainerOps int64 +} + +// ScoreIndex evaluates an IndexRecord against the scoring decision tree +// and returns an IndexAnalysis with the verdict populated. +func ScoreIndex(rec IndexRecord, cfg AnalysisConfig) IndexAnalysis { + now := cfg.Now + if now.IsZero() { + now = time.Now() + } + + ageDays := now.Sub(rec.AccessSince).Hours() / 24 + if ageDays < 0 { + ageDays = 0 + } + + var opsPerDay float64 + if ageDays > 0 { + opsPerDay = float64(rec.AccessOps) / ageDays + } + + var indexSizePct float64 + if rec.CollTotalIdxSize > 0 { + indexSizePct = float64(rec.IndexSizeBytes) / float64(rec.CollTotalIdxSize) * 100 + } + + a := IndexAnalysis{ + Namespace: rec.Namespace, + IndexName: rec.IndexName, + IndexKey: rec.IndexKey, + AccessOps: rec.AccessOps, + AccessSince: rec.AccessSince, + AgeDays: math.Round(ageDays*10) / 10, + OpsPerDay: math.Round(opsPerDay*10) / 10, + IndexSizeBytes: rec.IndexSizeBytes, + CollDocCount: rec.CollDocCount, + CollTotalIdxSize: rec.CollTotalIdxSize, + IndexSizePct: math.Round(indexSizePct*10) / 10, + IsPartial: rec.IsPartial, + IsSparse: rec.IsSparse, + IsUnique: rec.IsUnique, + IsTTL: rec.IsTTL, + IsHidden: rec.IsHidden, + WriteOpsPerSec: rec.WriteOpsPerSec, + } + + // Hard guards + if rec.IndexName == "_id_" { + a.Score = 0 + a.Recommendation = RecommendKeepConstraint + a.Confidence = ConfidenceHigh + a.Reason = "MongoDB required _id_ index" + return a + } + if rec.IsUnique { + a.Score = 0 + a.Recommendation = RecommendKeepConstraint + a.Confidence = ConfidenceHigh + a.Reason = "enforces uniqueness constraint" + return a + } + if rec.IsTTL { + a.Score = 0 + a.Recommendation = RecommendKeepConstraint + a.Confidence = ConfidenceHigh + a.Reason = "TTL index for automatic document expiration" + return a + } + if rec.IsHidden { + a.Score = 0 + a.Recommendation = RecommendKeepHidden + a.Confidence = ConfidenceHigh + a.Reason = "intentionally excluded from query planner by admin" + return a + } + + // Warmup period check + if ageDays < cfg.WarmupDays { + a.Score = 0.1 + a.Recommendation = RecommendMonitor + a.Confidence = ConfidenceLow + a.Reason = fmt.Sprintf("Index created/stats reset %.0f days ago; re-check after %.0f days", + ageDays, cfg.WarmupDays) + return a + } + + // Cross-reference with duplicate check + if cfg.CrossReferenceDuplicates && rec.IsDuplicatePrefix && rec.AccessOps == 0 && rec.DuplicateContainerOps > 0 { + a.Score = 0.95 + a.Recommendation = RecommendSafeToDrop + a.Confidence = ConfidenceHigh + a.Reason = fmt.Sprintf("Prefix of '%s' which is actively used (%d ops); this shorter index is redundant", + rec.DuplicateContainerName, rec.DuplicateContainerOps) + return a + } + + // Zero-access scoring + if rec.AccessOps == 0 { + if rec.IsPartial || rec.IsSparse { + a.Score = 0.4 + a.Recommendation = RecommendKeepPartial + a.Confidence = ConfidenceMedium + a.Reason = "Partial/sparse indexes may legitimately have low access; verify the filter expression matches current query patterns" + return a + } + if rec.CollDocCount == 0 { + a.Score = 0.2 + a.Recommendation = RecommendMonitor + a.Confidence = ConfidenceLow + a.Reason = "Collection is empty; index cannot have been used" + return a + } + if rec.IndexSizeBytes > cfg.LargeIndexSizeBytes { + a.Score = 0.95 + a.Recommendation = RecommendSafeToDrop + a.Confidence = ConfidenceHigh + a.Reason = fmt.Sprintf("Zero reads in %.0f days; index is %s and costs write amplification", + ageDays, FormatBytes(rec.IndexSizeBytes)) + return a + } + a.Score = 0.8 + a.Recommendation = RecommendLikelyUnused + a.Confidence = ConfidenceHigh + a.Reason = fmt.Sprintf("Zero reads in %.0f days; small index (%s), low cost to keep", + ageDays, FormatBytes(rec.IndexSizeBytes)) + return a + } + + // Low-usage scoring + if opsPerDay < cfg.LowUsageThreshold { + writesPerDay := math.Max(rec.WriteOpsPerSec*86400, 1) + usageRatio := opsPerDay / writesPerDay + + if usageRatio < 0.0001 { + a.Score = 0.7 + a.Recommendation = RecommendLowUsage + a.Confidence = ConfidenceMedium + a.Reason = fmt.Sprintf("Index used %.1f times/day on a collection with %.0f writes/day; read benefit negligible vs write cost", + opsPerDay, writesPerDay) + return a + } + a.Score = 0.5 + a.Recommendation = RecommendLowUsage + a.Confidence = ConfidenceMedium + a.Reason = fmt.Sprintf("Index used %.1f times/day; consider monitoring", opsPerDay) + return a + } + + // Index is actively used + a.Score = 0 + a.Recommendation = "" + a.Confidence = "" + a.Reason = "" + return a +} + +func FormatBytes(b int64) string { + const ( + kb = 1024 + mb = kb * 1024 + gb = mb * 1024 + ) + switch { + case b >= gb: + return fmt.Sprintf("%.1f GB", float64(b)/float64(gb)) + case b >= mb: + return fmt.Sprintf("%.1f MB", float64(b)/float64(mb)) + case b >= kb: + return fmt.Sprintf("%.1f KB", float64(b)/float64(kb)) + default: + return fmt.Sprintf("%d B", b) + } +} diff --git a/src/go/pt-mongodb-index-check/indexes/analysis_test.go b/src/go/pt-mongodb-index-check/indexes/analysis_test.go new file mode 100644 index 000000000..734c7f0af --- /dev/null +++ b/src/go/pt-mongodb-index-check/indexes/analysis_test.go @@ -0,0 +1,660 @@ +package indexes + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func baseConfig() AnalysisConfig { + return AnalysisConfig{ + WarmupDays: 7, + LowUsageThreshold: 1.0, + LargeIndexSizeBytes: 10 * 1024 * 1024, // 10 MB + IncludeLowUsage: true, + CrossReferenceDuplicates: true, + Now: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC), + } +} + +func TestScoreIndex(t *testing.T) { + cfg := baseConfig() + thirtyDaysAgo := cfg.Now.AddDate(0, 0, -30) + threeDaysAgo := cfg.Now.AddDate(0, 0, -3) + + tests := []struct { + name string + rec IndexRecord + wantRec string + wantScoreMin float64 + wantScoreMax float64 + wantConfidence string + }{ + { + name: "_id_ index is always kept", + rec: IndexRecord{ + IndexName: "_id_", + AccessOps: 0, + AccessSince: thirtyDaysAgo, + }, + wantRec: RecommendKeepConstraint, + wantScoreMin: 0, + wantScoreMax: 0, + wantConfidence: ConfidenceHigh, + }, + { + name: "unique index is always kept", + rec: IndexRecord{ + IndexName: "email_1", + AccessOps: 0, + AccessSince: thirtyDaysAgo, + IsUnique: true, + }, + wantRec: RecommendKeepConstraint, + wantScoreMin: 0, + wantScoreMax: 0, + wantConfidence: ConfidenceHigh, + }, + { + name: "TTL index is always kept", + rec: IndexRecord{ + IndexName: "createdAt_1", + AccessOps: 0, + AccessSince: thirtyDaysAgo, + IsTTL: true, + }, + wantRec: RecommendKeepConstraint, + wantScoreMin: 0, + wantScoreMax: 0, + wantConfidence: ConfidenceHigh, + }, + { + name: "hidden index is kept", + rec: IndexRecord{ + IndexName: "hidden_idx", + AccessOps: 0, + AccessSince: thirtyDaysAgo, + IsHidden: true, + }, + wantRec: RecommendKeepHidden, + wantScoreMin: 0, + wantScoreMax: 0, + wantConfidence: ConfidenceHigh, + }, + { + name: "warmup period - stats reset recently", + rec: IndexRecord{ + IndexName: "new_idx", + AccessOps: 0, + AccessSince: threeDaysAgo, + }, + wantRec: RecommendMonitor, + wantScoreMin: 0.1, + wantScoreMax: 0.1, + wantConfidence: ConfidenceLow, + }, + { + name: "zero ops, large index -> safe to drop", + rec: IndexRecord{ + IndexName: "old_big_idx", + AccessOps: 0, + AccessSince: thirtyDaysAgo, + IndexSizeBytes: 100 * 1024 * 1024, // 100 MB + CollDocCount: 1000, + }, + wantRec: RecommendSafeToDrop, + wantScoreMin: 0.95, + wantScoreMax: 0.95, + wantConfidence: ConfidenceHigh, + }, + { + name: "zero ops, small index -> likely unused", + rec: IndexRecord{ + IndexName: "old_small_idx", + AccessOps: 0, + AccessSince: thirtyDaysAgo, + IndexSizeBytes: 1024, // 1 KB + CollDocCount: 100, + }, + wantRec: RecommendLikelyUnused, + wantScoreMin: 0.8, + wantScoreMax: 0.8, + wantConfidence: ConfidenceHigh, + }, + { + name: "zero ops, partial index -> keep partial", + rec: IndexRecord{ + IndexName: "partial_idx", + AccessOps: 0, + AccessSince: thirtyDaysAgo, + IsPartial: true, + CollDocCount: 1000, + }, + wantRec: RecommendKeepPartial, + wantScoreMin: 0.4, + wantScoreMax: 0.4, + wantConfidence: ConfidenceMedium, + }, + { + name: "zero ops, sparse index -> keep partial", + rec: IndexRecord{ + IndexName: "sparse_idx", + AccessOps: 0, + AccessSince: thirtyDaysAgo, + IsSparse: true, + CollDocCount: 1000, + }, + wantRec: RecommendKeepPartial, + wantScoreMin: 0.4, + wantScoreMax: 0.4, + wantConfidence: ConfidenceMedium, + }, + { + name: "zero ops, empty collection -> monitor", + rec: IndexRecord{ + IndexName: "idx_on_empty", + AccessOps: 0, + AccessSince: thirtyDaysAgo, + CollDocCount: 0, + }, + wantRec: RecommendMonitor, + wantScoreMin: 0.2, + wantScoreMax: 0.2, + wantConfidence: ConfidenceLow, + }, + { + name: "low usage, high write cost", + rec: IndexRecord{ + IndexName: "low_use_idx", + AccessOps: 5, + AccessSince: thirtyDaysAgo, + IndexSizeBytes: 50 * 1024 * 1024, + CollDocCount: 100000, + WriteOpsPerSec: 1000, + }, + wantRec: RecommendLowUsage, + wantScoreMin: 0.5, + wantScoreMax: 0.7, + wantConfidence: ConfidenceMedium, + }, + { + name: "low usage, negligible ratio vs writes", + rec: IndexRecord{ + IndexName: "low_ratio_idx", + AccessOps: 1, + AccessSince: thirtyDaysAgo, + IndexSizeBytes: 50 * 1024 * 1024, + CollDocCount: 100000, + WriteOpsPerSec: 10000, + }, + wantRec: RecommendLowUsage, + wantScoreMin: 0.7, + wantScoreMax: 0.7, + wantConfidence: ConfidenceMedium, + }, + { + name: "duplicate cross-ref: prefix unused, container used", + rec: IndexRecord{ + IndexName: "prefix_idx", + AccessOps: 0, + AccessSince: thirtyDaysAgo, + CollDocCount: 1000, + IndexSizeBytes: 5000, + IsDuplicatePrefix: true, + DuplicateContainerName: "full_idx", + DuplicateContainerOps: 50000, + }, + wantRec: RecommendSafeToDrop, + wantScoreMin: 0.95, + wantScoreMax: 0.95, + wantConfidence: ConfidenceHigh, + }, + { + name: "actively used index - no recommendation", + rec: IndexRecord{ + IndexName: "active_idx", + AccessOps: 100000, + AccessSince: thirtyDaysAgo, + CollDocCount: 1000, + }, + wantRec: "", + wantScoreMin: 0, + wantScoreMax: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ScoreIndex(tt.rec, cfg) + assert.Equal(t, tt.wantRec, result.Recommendation, "recommendation mismatch") + assert.GreaterOrEqual(t, result.Score, tt.wantScoreMin, "score too low") + assert.LessOrEqual(t, result.Score, tt.wantScoreMax, "score too high") + if tt.wantConfidence != "" { + assert.Equal(t, tt.wantConfidence, result.Confidence, "confidence mismatch") + } + }) + } +} + +func TestCompatibleIndexProperties(t *testing.T) { + tests := []struct { + name string + shorter collectionIndex + longer collectionIndex + expected bool + }{ + { + name: "identical properties - compatible", + shorter: collectionIndex{Name: "a"}, + longer: collectionIndex{Name: "b"}, + expected: true, + }, + { + name: "both partial, same filter - compatible", + shorter: collectionIndex{ + Name: "a", + PartialFilter: primitive.M{"status": "active"}, + }, + longer: collectionIndex{ + Name: "b", + PartialFilter: primitive.M{"status": "active"}, + }, + expected: true, + }, + { + name: "both partial, different filter - NOT compatible", + shorter: collectionIndex{ + Name: "a", + PartialFilter: primitive.M{"status": "active"}, + }, + longer: collectionIndex{ + Name: "b", + PartialFilter: primitive.M{"status": "inactive"}, + }, + expected: false, + }, + { + name: "one partial, one not - NOT compatible", + shorter: collectionIndex{ + Name: "a", + PartialFilter: primitive.M{"status": "active"}, + }, + longer: collectionIndex{ + Name: "b", + }, + expected: false, + }, + { + name: "different sparse - NOT compatible", + shorter: collectionIndex{ + Name: "a", + Sparse: true, + }, + longer: collectionIndex{ + Name: "b", + Sparse: false, + }, + expected: false, + }, + { + name: "both sparse - compatible", + shorter: collectionIndex{ + Name: "a", + Sparse: true, + }, + longer: collectionIndex{ + Name: "b", + Sparse: true, + }, + expected: true, + }, + { + name: "different collation - NOT compatible", + shorter: collectionIndex{ + Name: "a", + Collation: primitive.M{"locale": "en"}, + }, + longer: collectionIndex{ + Name: "b", + Collation: primitive.M{"locale": "fr"}, + }, + expected: false, + }, + { + name: "one collation, one not - NOT compatible", + shorter: collectionIndex{ + Name: "a", + Collation: primitive.M{"locale": "en"}, + }, + longer: collectionIndex{ + Name: "b", + }, + expected: false, + }, + { + name: "same collation - compatible", + shorter: collectionIndex{ + Name: "a", + Collation: primitive.M{"locale": "en"}, + }, + longer: collectionIndex{ + Name: "b", + Collation: primitive.M{"locale": "en"}, + }, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := compatibleIndexProperties(tt.shorter, tt.longer) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestFormatBytes(t *testing.T) { + tests := []struct { + input int64 + expected string + }{ + {0, "0 B"}, + {512, "512 B"}, + {1024, "1.0 KB"}, + {1536, "1.5 KB"}, + {1048576, "1.0 MB"}, + {10485760, "10.0 MB"}, + {1073741824, "1.0 GB"}, + } + + for _, tt := range tests { + t.Run(tt.expected, func(t *testing.T) { + assert.Equal(t, tt.expected, FormatBytes(tt.input)) + }) + } +} + +func TestBuildIndexRecords(t *testing.T) { + now := time.Now() + stats := []IndexStat{ + { + Name: "idx_a", + Key: primitive.D{{Key: "a", Value: int32(1)}}, + }, + } + stats[0].Accesses.Ops = 100 + stats[0].Accesses.Since = primitive.NewDateTimeFromTime(now.AddDate(0, 0, -10)) + + meta := map[string]indexMetadata{ + "idx_a": { + Name: "idx_a", + Unique: true, + }, + } + + cs := collectionStats{ + Count: 5000, + TotalIndexSize: 1000000, + IndexSizes: map[string]int64{"idx_a": 50000}, + } + + wr := serverWriteRate{WriteOpsPerSec: 200} + + records := BuildIndexRecords(stats, meta, cs, wr, "testdb.testcol") + + assert.Len(t, records, 1) + rec := records[0] + assert.Equal(t, "testdb.testcol", rec.Namespace) + assert.Equal(t, "idx_a", rec.IndexName) + assert.Equal(t, int64(100), rec.AccessOps) + assert.Equal(t, int64(50000), rec.IndexSizeBytes) + assert.Equal(t, int64(5000), rec.CollDocCount) + assert.True(t, rec.IsUnique) + assert.Equal(t, float64(200), rec.WriteOpsPerSec) +} + +func TestAggregateShardStats(t *testing.T) { + since1 := primitive.NewDateTimeFromTime(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + since2 := primitive.NewDateTimeFromTime(time.Date(2024, 1, 5, 0, 0, 0, 0, time.UTC)) + since3 := primitive.NewDateTimeFromTime(time.Date(2024, 1, 10, 0, 0, 0, 0, time.UTC)) + + stats := []IndexStat{ + {Name: "idx_a", Key: primitive.D{{Key: "a", Value: int32(1)}}, Host: "shard1:27018"}, + {Name: "idx_a", Key: primitive.D{{Key: "a", Value: int32(1)}}, Host: "shard2:27018"}, + {Name: "idx_a", Key: primitive.D{{Key: "a", Value: int32(1)}}, Host: "shard3:27018"}, + {Name: "idx_b", Key: primitive.D{{Key: "b", Value: int32(1)}}, Host: "shard1:27018"}, + } + stats[0].Accesses.Ops = 100 + stats[0].Accesses.Since = since1 + stats[1].Accesses.Ops = 200 + stats[1].Accesses.Since = since2 + stats[2].Accesses.Ops = 50 + stats[2].Accesses.Since = since3 + stats[3].Accesses.Ops = 10 + stats[3].Accesses.Since = since2 + + merged := AggregateShardStats(stats) + + assert.Len(t, merged, 2) + + assert.Equal(t, "idx_a", merged[0].Name) + assert.Equal(t, int64(350), merged[0].Accesses.Ops) + assert.Equal(t, since1, merged[0].Accesses.Since, "should use the oldest since") + assert.Equal(t, 3, merged[0].ShardCount) + + assert.Equal(t, "idx_b", merged[1].Name) + assert.Equal(t, int64(10), merged[1].Accesses.Ops) + assert.Equal(t, 1, merged[1].ShardCount) +} + +func TestAggregateShardStats_singleShard(t *testing.T) { + since := primitive.NewDateTimeFromTime(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC)) + stats := []IndexStat{ + {Name: "idx_x", Key: primitive.D{{Key: "x", Value: int32(1)}}, Host: "primary:27017"}, + {Name: "idx_y", Key: primitive.D{{Key: "y", Value: int32(-1)}}, Host: "primary:27017"}, + } + stats[0].Accesses.Ops = 42 + stats[0].Accesses.Since = since + stats[1].Accesses.Ops = 99 + stats[1].Accesses.Since = since + + merged := AggregateShardStats(stats) + + assert.Len(t, merged, 2) + assert.Equal(t, int64(42), merged[0].Accesses.Ops) + assert.Equal(t, 1, merged[0].ShardCount) + assert.Equal(t, int64(99), merged[1].Accesses.Ops) + assert.Equal(t, 1, merged[1].ShardCount) +} + +func TestComparableKey_hashed(t *testing.T) { + idx := collectionIndex{ + Name: "_id_hashed", + Key: primitive.D{{Key: "_id", Value: "hashed"}}, + } + assert.Equal(t, "hashed:_id", idx.ComparableKey()) + + idIdx := collectionIndex{ + Name: "_id_", + Key: primitive.D{{Key: "_id", Value: int32(1)}}, + } + assert.Equal(t, "+_id", idIdx.ComparableKey()) + assert.NotEqual(t, idIdx.ComparableKey(), idx.ComparableKey(), + "_id_ and _id_hashed must produce different comparable keys") +} + +func TestComparableKey_text(t *testing.T) { + idx := collectionIndex{ + Name: "content_text", + Key: primitive.D{{Key: "content", Value: "text"}}, + } + assert.Equal(t, "text:content", idx.ComparableKey()) + + btree := collectionIndex{ + Name: "content_1", + Key: primitive.D{{Key: "content", Value: int32(1)}}, + } + assert.NotEqual(t, btree.ComparableKey(), idx.ComparableKey(), + "text index must not match B-tree index on same field") +} + +func TestComparableKey_2dsphere(t *testing.T) { + idx := collectionIndex{ + Name: "location_2dsphere", + Key: primitive.D{{Key: "location", Value: "2dsphere"}}, + } + assert.Equal(t, "2dsphere:location", idx.ComparableKey()) +} + +func TestComparableKey_int64Direction(t *testing.T) { + idx := collectionIndex{ + Name: "a_1", + Key: primitive.D{{Key: "a", Value: int64(1)}}, + } + assert.Equal(t, "+a", idx.ComparableKey()) +} + +func TestNormalizeIndexStat(t *testing.T) { + s := IndexStat{} + s.Spec.Name = "my_idx" + s.Spec.Key = primitive.D{{Key: "x", Value: int32(1)}} + NormalizeIndexStat(&s) + assert.Equal(t, "my_idx", s.Name) + assert.Equal(t, primitive.D{{Key: "x", Value: int32(1)}}, s.Key) +} + +func TestDeduplicateIndexRecords(t *testing.T) { + t1 := time.Date(2024, 1, 10, 0, 0, 0, 0, time.UTC) + t2 := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + recs := []IndexRecord{ + {IndexName: "idx_a", AccessOps: 10, AccessSince: t1}, + {IndexName: "idx_a", AccessOps: 5, AccessSince: t2}, + {IndexName: "idx_b", AccessOps: 3, AccessSince: t1}, + } + out := DeduplicateIndexRecords(recs) + assert.Len(t, out, 2) + assert.Equal(t, "idx_a", out[0].IndexName) + assert.Equal(t, int64(15), out[0].AccessOps) + assert.Equal(t, t2, out[0].AccessSince) + assert.Equal(t, "idx_b", out[1].IndexName) +} + +func TestAggregateShardStats_afterNormalize(t *testing.T) { + since1 := primitive.NewDateTimeFromTime(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + since2 := primitive.NewDateTimeFromTime(time.Date(2024, 1, 5, 0, 0, 0, 0, time.UTC)) + + a := IndexStat{Name: "", Host: "s1"} + a.Spec.Name = "same_idx" + a.Accesses.Ops = 1 + a.Accesses.Since = since1 + b := IndexStat{Name: "", Host: "s2"} + b.Spec.Name = "same_idx" + b.Accesses.Ops = 2 + b.Accesses.Since = since2 + + NormalizeIndexStat(&a) + NormalizeIndexStat(&b) + merged := AggregateShardStats([]IndexStat{a, b}) + + assert.Len(t, merged, 1) + assert.Equal(t, "same_idx", merged[0].Name) + assert.Equal(t, int64(3), merged[0].Accesses.Ops) + assert.Equal(t, since1, merged[0].Accesses.Since) + assert.Equal(t, 2, merged[0].ShardCount) +} + +func TestFindDuplicated_skips_id(t *testing.T) { + idIdx := collectionIndex{ + Name: "_id_", + Key: primitive.D{{Key: "_id", Value: int32(1)}}, + } + idHashedIdx := collectionIndex{ + Name: "_id_hashed", + Key: primitive.D{{Key: "_id", Value: "hashed"}}, + } + compoundIdx := collectionIndex{ + Name: "id_status", + Key: primitive.D{{Key: "_id", Value: int32(1)}, {Key: "status", Value: int32(1)}}, + } + + _ = idIdx + _ = idHashedIdx + _ = compoundIdx + // _id_ has ComparableKey "+_id" which is a prefix of "+_id+status" + // Without the _id_ skip, _id_ would be flagged as duplicate of id_status. + // We can't call FindDuplicated directly (needs a real DB), but we can + // verify that _id_hashed != _id_ via ComparableKey. + assert.NotEqual(t, "+_id", "hashed:_id") +} + +func TestFindDuplicated_hashed_not_prefix(t *testing.T) { + btreeKey := collectionIndex{ + Name: "field_1", + Key: primitive.D{{Key: "field", Value: int32(1)}}, + } + hashedKey := collectionIndex{ + Name: "field_hashed", + Key: primitive.D{{Key: "field", Value: "hashed"}}, + } + assert.False(t, + strings.HasPrefix(hashedKey.ComparableKey(), btreeKey.ComparableKey()), + "hashed index should not be a prefix of B-tree index") + assert.False(t, + strings.HasPrefix(btreeKey.ComparableKey(), hashedKey.ComparableKey()), + "B-tree index should not be a prefix of hashed index") +} + +// TestComparableKey_hashedSymbol ensures that a key value encoded as +// primitive.Symbol("hashed") — as seen on MongoDB 8.x — produces a different +// ComparableKey than a plain B-tree ascending key, preventing false positive +// duplicate detection between {phone:1} and {phone:"hashed"}. +func TestComparableKey_hashedSymbol(t *testing.T) { + btree := collectionIndex{ + Name: "phone_1", + Key: primitive.D{{Key: "phone", Value: int32(1)}}, + } + hashedSymbol := collectionIndex{ + Name: "phone_hashed", + Key: primitive.D{{Key: "phone", Value: primitive.Symbol("hashed")}}, + } + + assert.Equal(t, "+phone", btree.ComparableKey()) + assert.Equal(t, "hashed:phone", hashedSymbol.ComparableKey()) + assert.False(t, + strings.HasPrefix(hashedSymbol.ComparableKey(), btree.ComparableKey()), + "hashed (Symbol) index should not be a prefix of B-tree index") + assert.False(t, + strings.HasPrefix(btree.ComparableKey(), hashedSymbol.ComparableKey()), + "B-tree index should not be a prefix of hashed (Symbol) index") +} + +func TestCrossReferenceDuplicates(t *testing.T) { + records := []IndexRecord{ + {IndexName: "idx_short", AccessOps: 0}, + {IndexName: "idx_full", AccessOps: 5000}, + } + + duplicates := []Duplicate{ + { + Name: "idx_short", + ContainerName: "idx_full", + }, + } + + allStats := []IndexStat{ + {Name: "idx_short"}, + {Name: "idx_full"}, + } + allStats[0].Accesses.Ops = 0 + allStats[1].Accesses.Ops = 5000 + + CrossReferenceDuplicates(records, duplicates, allStats) + + assert.True(t, records[0].IsDuplicatePrefix) + assert.Equal(t, "idx_full", records[0].DuplicateContainerName) + assert.Equal(t, int64(5000), records[0].DuplicateContainerOps) + assert.False(t, records[1].IsDuplicatePrefix) +} diff --git a/src/go/pt-mongodb-index-check/indexes/collector.go b/src/go/pt-mongodb-index-check/indexes/collector.go new file mode 100644 index 000000000..ca57aad61 --- /dev/null +++ b/src/go/pt-mongodb-index-check/indexes/collector.go @@ -0,0 +1,246 @@ +package indexes + +import ( + "context" + + "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" +) + +// indexMetadata holds index properties from listIndexes that $indexStats doesn't provide. +type indexMetadata struct { + Name string `bson:"name"` + Key primitive.D `bson:"key"` + Unique bool `bson:"unique,omitempty"` + Sparse bool `bson:"sparse,omitempty"` + PartialFilterExpression primitive.M `bson:"partialFilterExpression,omitempty"` + ExpireAfterSeconds *int32 `bson:"expireAfterSeconds,omitempty"` + Hidden bool `bson:"hidden,omitempty"` +} + +type collectionStats struct { + Count int64 `bson:"count"` + Size int64 `bson:"size"` + TotalIndexSize int64 `bson:"totalIndexSize"` + IndexSizes map[string]int64 `bson:"indexSizes"` +} + +type serverWriteRate struct { + WriteOpsPerSec float64 + Uptime int64 +} + +// CollectIndexMetadata retrieves index properties from listIndexes for a collection. +func CollectIndexMetadata(ctx context.Context, client *mongo.Client, database, collection string) (map[string]indexMetadata, error) { + cursor, err := client.Database(database).Collection(collection).Indexes().List(ctx, nil) + if err != nil { + return nil, errors.Wrap(err, "cannot list indexes") + } + + result := make(map[string]indexMetadata) + var indexes []indexMetadata + if err := cursor.All(ctx, &indexes); err != nil { + return nil, errors.Wrap(err, "cannot decode index metadata") + } + for _, idx := range indexes { + result[idx.Name] = idx + } + return result, nil +} + +// CollectCollStats retrieves collection statistics via the collStats command. +func CollectCollStats(ctx context.Context, client *mongo.Client, database, collection string) (collectionStats, error) { + var stats collectionStats + res := client.Database(database).RunCommand(ctx, bson.D{{Key: "collStats", Value: collection}}) + if err := res.Err(); err != nil { + return stats, errors.Wrap(err, "cannot run collStats") + } + if err := res.Decode(&stats); err != nil { + return stats, errors.Wrap(err, "cannot decode collStats") + } + return stats, nil +} + +// CollectServerWriteRate retrieves the global write rate from serverStatus opcounters. +func CollectServerWriteRate(ctx context.Context, client *mongo.Client) (serverWriteRate, error) { + var result struct { + Uptime int64 `bson:"uptime"` + OpCounters struct { + Insert int64 `bson:"insert"` + Update int64 `bson:"update"` + Delete int64 `bson:"delete"` + } `bson:"opcounters"` + } + + res := client.Database("admin").RunCommand(ctx, bson.D{{Key: "serverStatus", Value: 1}}) + if err := res.Err(); err != nil { + return serverWriteRate{}, errors.Wrap(err, "cannot run serverStatus") + } + if err := res.Decode(&result); err != nil { + return serverWriteRate{}, errors.Wrap(err, "cannot decode serverStatus") + } + + totalWrites := result.OpCounters.Insert + result.OpCounters.Update + result.OpCounters.Delete + var wps float64 + if result.Uptime > 0 { + wps = float64(totalWrites) / float64(result.Uptime) + } + + return serverWriteRate{ + WriteOpsPerSec: wps, + Uptime: result.Uptime, + }, nil +} + +// CollectIndexStats retrieves $indexStats for all indexes on a collection. +// Unlike FindUnused, this returns ALL indexes (not just ops==0) for the scoring engine. +func CollectIndexStats(ctx context.Context, client *mongo.Client, database, collection string) ([]IndexStat, error) { + aggregation := mongo.Pipeline{ + {{Key: "$indexStats", Value: primitive.M{}}}, + } + + cursor, err := client.Database(database).Collection(collection).Aggregate(ctx, aggregation) + if err != nil { + return nil, errors.Wrap(err, "cannot run $indexStats") + } + + var stats []IndexStat + if err = cursor.All(ctx, &stats); err != nil { + return nil, errors.Wrap(err, "cannot decode $indexStats") + } + for i := range stats { + NormalizeIndexStat(&stats[i]) + } + return stats, nil +} + +// AggregateShardStats deduplicates $indexStats entries that appear once per +// shard on sharded clusters. It groups by index name, sums ops across shards, +// and uses the oldest accesses.since as the observation window. +func AggregateShardStats(stats []IndexStat) []IndexStat { + type group struct { + merged IndexStat + count int + } + + groups := make(map[string]*group, len(stats)) + order := make([]string, 0, len(stats)) + + for _, s := range stats { + g, ok := groups[s.Name] + if !ok { + merged := s + merged.ShardCount = 1 + groups[s.Name] = &group{merged: merged, count: 1} + order = append(order, s.Name) + continue + } + g.count++ + g.merged.ShardCount = g.count + g.merged.Accesses.Ops += s.Accesses.Ops + if s.Accesses.Since < g.merged.Accesses.Since { + g.merged.Accesses.Since = s.Accesses.Since + } + } + + result := make([]IndexStat, 0, len(groups)) + for _, name := range order { + result = append(result, groups[name].merged) + } + return result +} + +// DeduplicateIndexRecords merges duplicate rows for the same index name (e.g. +// if stats were not aggregated) by summing ops and using the oldest AccessSince. +func DeduplicateIndexRecords(records []IndexRecord) []IndexRecord { + if len(records) <= 1 { + return records + } + byName := make(map[string]IndexRecord, len(records)) + order := make([]string, 0, len(records)) + for _, r := range records { + prev, ok := byName[r.IndexName] + if !ok { + byName[r.IndexName] = r + order = append(order, r.IndexName) + continue + } + prev.AccessOps += r.AccessOps + if !r.AccessSince.IsZero() && (prev.AccessSince.IsZero() || r.AccessSince.Before(prev.AccessSince)) { + prev.AccessSince = r.AccessSince + } + byName[r.IndexName] = prev + } + out := make([]IndexRecord, 0, len(order)) + for _, name := range order { + out = append(out, byName[name]) + } + return out +} + +// BuildIndexRecords merges data from $indexStats, listIndexes, collStats, and +// serverStatus into a slice of IndexRecord ready for scoring. +func BuildIndexRecords( + stats []IndexStat, + metadata map[string]indexMetadata, + cs collectionStats, + wr serverWriteRate, + namespace string, +) []IndexRecord { + records := make([]IndexRecord, 0, len(stats)) + + for _, s := range stats { + rec := IndexRecord{ + Namespace: namespace, + IndexName: s.Name, + IndexKey: s.Key, + AccessOps: s.Accesses.Ops, + AccessSince: s.Accesses.Since.Time(), + CollDocCount: cs.Count, + CollTotalIdxSize: cs.TotalIndexSize, + WriteOpsPerSec: wr.WriteOpsPerSec, + } + + if size, ok := cs.IndexSizes[s.Name]; ok { + rec.IndexSizeBytes = size + } + + if meta, ok := metadata[s.Name]; ok { + rec.IsPartial = len(meta.PartialFilterExpression) > 0 + rec.IsSparse = meta.Sparse + rec.IsUnique = meta.Unique + rec.IsTTL = meta.ExpireAfterSeconds != nil + rec.IsHidden = meta.Hidden + } + + records = append(records, rec) + } + + return records +} + +// CrossReferenceDuplicates annotates IndexRecords with duplicate prefix +// information by looking up each record in the duplicate results. +func CrossReferenceDuplicates(records []IndexRecord, duplicates []Duplicate, allStats []IndexStat) { + opsMap := make(map[string]int64, len(allStats)) + for _, s := range allStats { + opsMap[s.Name] += s.Accesses.Ops + } + + dupMap := make(map[string]Duplicate, len(duplicates)) + for _, d := range duplicates { + dupMap[d.Name] = d + } + + for i := range records { + if dup, ok := dupMap[records[i].IndexName]; ok { + records[i].IsDuplicatePrefix = true + records[i].DuplicateContainerName = dup.ContainerName + if ops, ok := opsMap[dup.ContainerName]; ok { + records[i].DuplicateContainerOps = ops + } + } + } +} diff --git a/src/go/pt-mongodb-index-check/indexes/duplicated.go b/src/go/pt-mongodb-index-check/indexes/duplicated.go index 89742e225..9fce26489 100644 --- a/src/go/pt-mongodb-index-check/indexes/duplicated.go +++ b/src/go/pt-mongodb-index-check/indexes/duplicated.go @@ -2,38 +2,106 @@ package indexes import ( "context" - "log" + "fmt" + "reflect" "sort" "strings" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" ) type collectionIndex struct { - Name string `bson:"name"` - Namespace string `bson:"ns"` - V int `bson:"v"` - Key primitive.D `bson:"key"` + Name string `bson:"name"` + Namespace string `bson:"ns"` + V int `bson:"v"` + Key primitive.D `bson:"key"` + PartialFilter primitive.M `bson:"partialFilterExpression,omitempty"` + Sparse bool `bson:"sparse,omitempty"` + Unique bool `bson:"unique,omitempty"` + Collation primitive.M `bson:"collation,omitempty"` } func (di collectionIndex) ComparableKey() string { str := "" for _, elem := range di.Key { - str += sign(elem) + elem.Key + str += keyToken(elem) } return str } +// keyToken produces a unique prefix token for an index key element. +// Numeric directions map to "+" or "-", while string index types +// (hashed, text, 2dsphere, 2d) use "type:" so they never collide +// with B-tree directions. +// +// MongoDB 8.x may encode the "hashed" key value as BSON Symbol (type 0x0E) +// rather than a plain BSON String (type 0x02). primitive.Symbol is a distinct +// Go named type (type Symbol string) so it does not match case string and +// requires its own branch. +func keyToken(elem primitive.E) string { + log.Debugf("keyToken field=%q type=%T value=%v", elem.Key, elem.Value, elem.Value) + switch v := elem.Value.(type) { + case int32: + if v < 0 { + return "-" + elem.Key + } + return "+" + elem.Key + case int64: + if v < 0 { + return "-" + elem.Key + } + return "+" + elem.Key + case float64: + if v < 0 { + return "-" + elem.Key + } + return "+" + elem.Key + case string: + switch v { + case "hashed", "text", "2dsphere", "2d": + return v + ":" + elem.Key + default: + return "+" + elem.Key + } + case primitive.Symbol: + s := string(v) + switch s { + case "hashed", "text", "2dsphere", "2d": + return s + ":" + elem.Key + default: + return "+" + elem.Key + } + case []byte: + s := string(v) + switch s { + case "hashed", "text", "2dsphere", "2d": + return s + ":" + elem.Key + default: + return "+" + elem.Key + } + default: + s := fmt.Sprint(elem.Value) + switch s { + case "hashed", "text", "2dsphere", "2d": + return s + ":" + elem.Key + default: + return "+" + elem.Key + } + } +} + func sign(elem primitive.E) string { sign := "+" - switch elem.Value.(type) { - case int32: // internal MongoDB indexes like _id_ or lastUsed have the sign field as int32. - if elem.Value.(int32) < 0 { + switch v := elem.Value.(type) { + case int32: + if v < 0 { sign = "-" } - case float64: // All other indexes have the sign field as float64. - if elem.Value.(float64) < 0 { + case float64: + if v < 0 { sign = "-" } } @@ -63,6 +131,37 @@ type Duplicate struct { Key IndexKey ContainerName string ContainerKey IndexKey + Warning string `json:",omitempty"` +} + +// compatibleIndexProperties returns true if two indexes have compatible +// properties for a prefix-duplicate relationship. Indexes with different +// partialFilterExpression, sparse settings, or collation serve different +// purposes and should not be considered duplicates. +func compatibleIndexProperties(shorter, longer collectionIndex) bool { + hasPartialI := len(shorter.PartialFilter) > 0 + hasPartialJ := len(longer.PartialFilter) > 0 + if hasPartialI != hasPartialJ { + return false + } + if hasPartialI && hasPartialJ && !reflect.DeepEqual(shorter.PartialFilter, longer.PartialFilter) { + return false + } + + if shorter.Sparse != longer.Sparse { + return false + } + + hasCollI := len(shorter.Collation) > 0 + hasCollJ := len(longer.Collation) > 0 + if hasCollI != hasCollJ { + return false + } + if hasCollI && hasCollJ && !reflect.DeepEqual(shorter.Collation, longer.Collation) { + return false + } + + return true } func FindDuplicated(ctx context.Context, client *mongo.Client, database, collection string) ([]Duplicate, error) { @@ -74,8 +173,8 @@ func FindDuplicated(ctx context.Context, client *mongo.Client, database, collect } var results []collectionIndex - if err = cursor.All(context.TODO(), &results); err != nil { - log.Fatal(err) + if err = cursor.All(ctx, &results); err != nil { + return nil, errors.Wrap(err, "cannot decode index list") } sort.Slice(results, func(i, j int) bool { @@ -83,19 +182,36 @@ func FindDuplicated(ctx context.Context, client *mongo.Client, database, collect }) for i := 0; i < len(results)-1; i++ { + if results[i].Name == "_id_" { + continue + } for j := i + 1; j < len(results); j++ { - if strings.HasPrefix(results[j].ComparableKey(), results[i].ComparableKey()) { - idx := Duplicate{ - Namespace: database + "." + collection, - Name: results[i].Name, - Key: make([]primitive.E, len(results[i].Key)), - ContainerName: results[j].Name, - ContainerKey: make([]primitive.E, len(results[j].Key)), - } - copy(idx.Key, results[i].Key) - copy(idx.ContainerKey, results[j].Key) - di = append(di, idx) + ki, kj := results[i].ComparableKey(), results[j].ComparableKey() + if ki == kj { + continue } + if !strings.HasPrefix(kj, ki) { + continue + } + if !compatibleIndexProperties(results[i], results[j]) { + continue + } + + idx := Duplicate{ + Namespace: database + "." + collection, + Name: results[i].Name, + Key: make([]primitive.E, len(results[i].Key)), + ContainerName: results[j].Name, + ContainerKey: make([]primitive.E, len(results[j].Key)), + } + copy(idx.Key, results[i].Key) + copy(idx.ContainerKey, results[j].Key) + + if results[i].Unique && !results[j].Unique { + idx.Warning = "prefix index enforces unique constraint; dropping requires the container index to also be unique" + } + + di = append(di, idx) } } diff --git a/src/go/pt-mongodb-index-check/indexes/unused.go b/src/go/pt-mongodb-index-check/indexes/unused.go index c506b8d1c..bff82f11e 100644 --- a/src/go/pt-mongodb-index-check/indexes/unused.go +++ b/src/go/pt-mongodb-index-check/indexes/unused.go @@ -2,11 +2,11 @@ package indexes import ( "context" + "strings" "github.com/pkg/errors" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" - "gopkg.in/mgo.v2/bson" ) var systemDBs = []string{"admin", "config", "local", "system.profile"} //nolint:gochecknoglobals @@ -22,9 +22,22 @@ type IndexStat struct { V int32 `bson:"v"` Key primitive.D `bson:"key"` } `bson:"spec"` - Name string `bson:"name"` - Key primitive.D `bson:"key"` - Host string `bson:"host"` + Name string `bson:"name"` + Key primitive.D `bson:"key"` + Host string `bson:"host"` + ShardCount int `bson:"-"` +} + +// NormalizeIndexStat fills top-level name/key from spec when the driver or +// mongos omits them on $indexStats documents, so aggregation and lookups +// group by the correct index name. +func NormalizeIndexStat(s *IndexStat) { + if s.Name == "" && s.Spec.Name != "" { + s.Name = s.Spec.Name + } + if len(s.Key) == 0 && len(s.Spec.Key) > 0 { + s.Key = s.Spec.Key + } } func in(search string, items []string) bool { @@ -36,12 +49,23 @@ func in(search string, items []string) bool { return false } +// IsSystemDB returns true if the database is a MongoDB system database +// that should be skipped during index analysis. +func IsSystemDB(database string) bool { + return in(database, systemDBs) +} + +// IsSystemCollection returns true for MongoDB internal collections whose +// names start with "system." (e.g. system.profile, system.js). These should +// be skipped alongside system databases (admin, config, local). +func IsSystemCollection(collection string) bool { + return strings.HasPrefix(collection, "system.") +} + // FindUnusedIndexes returns a list of unused indexes for the given database and collection. func FindUnused(ctx context.Context, client *mongo.Client, database, collection string) ([]IndexStat, error) { aggregation := mongo.Pipeline{ {{Key: "$indexStats", Value: primitive.M{}}}, - {{Key: "$match", Value: primitive.M{"accesses.ops": 0}}}, - {{Key: "$match", Value: primitive.M{"name": bson.M{"$ne": "_id_"}}}}, } if in(database, systemDBs) { @@ -57,6 +81,20 @@ func FindUnused(ctx context.Context, client *mongo.Client, database, collection if err = cursor.All(ctx, &stats); err != nil { return nil, errors.Wrap(err, "cannot get $indexStats for unused indexes") } + for i := range stats { + NormalizeIndexStat(&stats[i]) + } + stats = AggregateShardStats(stats) - return stats, nil + var out []IndexStat + for _, s := range stats { + if s.Name == "_id_" { + continue + } + if s.Accesses.Ops != 0 { + continue + } + out = append(out, s) + } + return out, nil } diff --git a/src/go/pt-mongodb-index-check/indexes/unused_test.go b/src/go/pt-mongodb-index-check/indexes/unused_test.go index 8a0ff66cd..7cc2bbccb 100644 --- a/src/go/pt-mongodb-index-check/indexes/unused_test.go +++ b/src/go/pt-mongodb-index-check/indexes/unused_test.go @@ -80,3 +80,17 @@ func TestUnusedIndexes(t *testing.T) { assert.Equal(t, want, got) } + +func TestIsSystemDB(t *testing.T) { + assert.True(t, IsSystemDB("admin")) + assert.True(t, IsSystemDB("config")) + assert.True(t, IsSystemDB("local")) + assert.False(t, IsSystemDB("myapp")) +} + +func TestIsSystemCollection(t *testing.T) { + assert.True(t, IsSystemCollection("system.profile")) + assert.True(t, IsSystemCollection("system.js")) + assert.False(t, IsSystemCollection("orders")) + assert.False(t, IsSystemCollection("users")) +} diff --git a/src/go/pt-mongodb-index-check/main.go b/src/go/pt-mongodb-index-check/main.go index 24cc05e07..ff8de9fff 100644 --- a/src/go/pt-mongodb-index-check/main.go +++ b/src/go/pt-mongodb-index-check/main.go @@ -5,6 +5,8 @@ import ( "context" "encoding/json" "fmt" + "net/url" + "os" "strings" "text/template" "time" @@ -15,6 +17,7 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readpref" "github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/indexes" "github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/templates" @@ -34,11 +37,149 @@ type cmdlineArgs struct { Collections []string `name:"collections" xor:"colls" help:"Comma separated list of collections to check"` URI string `name:"mongodb.uri" required:"" placeholder:"mongodb://host:port/admindb?options" help:"Connection URI"` JSON bool `name:"json" help:"Show output as JSON"` + + WarmupDays float64 `name:"warmup-days" default:"7" help:"Minimum observation window (days) before flagging unused indexes"` + LowUsageThreshold float64 `name:"low-usage-threshold" default:"1.0" help:"Ops/day below which an index is considered low-usage"` + LargeIndexSize int64 `name:"large-index-size" default:"10485760" help:"Index size threshold in bytes for 'large' classification (default 10MB)"` + IncludeLowUsage bool `name:"include-low-usage" default:"false" help:"Also report indexes with low but non-zero usage"` + CrossReferenceDuplicates bool `name:"cross-reference-duplicates" default:"true" help:"Combine unused + duplicate analysis for better recommendations"` } type response struct { - Unused []indexes.IndexStat - Duplicated []indexes.Duplicate + Unused []indexes.IndexStat `json:"Unused,omitempty"` + Duplicated []indexes.Duplicate `json:"Duplicated,omitempty"` + Analysis []indexes.IndexAnalysis `json:"Analysis,omitempty"` +} + +// analysisReportData is the template data for the sectioned analysis report. +type analysisReportData struct { + ObsStart string + ObsEnd string + ObsDays string + TotalAnalyzed int + DatabaseCount int + CollectionCount int + WriteRate string + + SafeToDrop []indexes.IndexAnalysis + LikelyUnused []indexes.IndexAnalysis + LowUsage []indexes.IndexAnalysis + Monitor []indexes.IndexAnalysis + Keep []indexes.IndexAnalysis + + SafeToDropCount int + SafeToDropSavings int64 + LikelyUnusedCount int + LowUsageCount int + MonitorCount int + KeepCount int +} + +// duplicateDisplayRow holds per-pair data for the rich duplicate text report. +type duplicateDisplayRow struct { + Namespace string + Name string + Key indexes.IndexKey + ContainerName string + ContainerKey indexes.IndexKey + Warning string + Reason string + Action string + PrefixSize int64 + ContainerSize int64 + HasSizes bool +} + +// duplicateReportData is the template data for the sectioned duplicate report. +type duplicateReportData struct { + TotalPairs int + DatabaseCount int + CollectionCount int + Standard []duplicateDisplayRow + WithWarning []duplicateDisplayRow +} + +// buildDuplicateReportData transforms a raw []Duplicate into the display struct +// used by renderDuplicateReport. sizeByNS maps namespace → index name → size +// in bytes; pass nil to skip size display. +func buildDuplicateReportData(dups []indexes.Duplicate, sizeByNS map[string]map[string]int64) duplicateReportData { + dbSet := make(map[string]struct{}) + nsSet := make(map[string]struct{}) + for _, d := range dups { + nsSet[d.Namespace] = struct{}{} + parts := strings.SplitN(d.Namespace, ".", 2) + dbSet[parts[0]] = struct{}{} + } + + data := duplicateReportData{ + TotalPairs: len(dups), + DatabaseCount: len(dbSet), + CollectionCount: len(nsSet), + } + + collNameFn := func(ns string) string { + parts := strings.SplitN(ns, ".", 2) + if len(parts) == 2 { + return parts[1] + } + return ns + } + + for _, d := range dups { + reason := fmt.Sprintf("'%s' is a key-order prefix of '%s'; any query served by '%s' can also use '%s'.", + d.Name, d.ContainerName, d.Name, d.ContainerName) + action := fmt.Sprintf("db.%s.dropIndex(%q)", collNameFn(d.Namespace), d.Name) + + row := duplicateDisplayRow{ + Namespace: d.Namespace, + Name: d.Name, + Key: d.Key, + ContainerName: d.ContainerName, + ContainerKey: d.ContainerKey, + Warning: d.Warning, + Reason: reason, + Action: action, + } + + if sizes, ok := sizeByNS[d.Namespace]; ok { + row.PrefixSize = sizes[d.Name] + row.ContainerSize = sizes[d.ContainerName] + row.HasSizes = true + } + + if d.Warning == "" { + data.Standard = append(data.Standard, row) + } else { + data.WithWarning = append(data.WithWarning, row) + } + } + + return data +} + +// collectDuplicateSizes fetches collStats for each unique namespace in the +// duplicate list and returns a map of namespace → index name → size in bytes. +// Errors per namespace are logged as warnings and that namespace is omitted. +func collectDuplicateSizes(ctx context.Context, client *mongo.Client, dups []indexes.Duplicate) map[string]map[string]int64 { + nsSet := make(map[string]struct{}) + for _, d := range dups { + nsSet[d.Namespace] = struct{}{} + } + + result := make(map[string]map[string]int64, len(nsSet)) + for ns := range nsSet { + parts := strings.SplitN(ns, ".", 2) + if len(parts) != 2 { + continue + } + cs, err := indexes.CollectCollStats(ctx, client, parts[0], parts[1]) + if err != nil { + log.Warnf("cannot get collStats for %s (sizes omitted from duplicate report): %s", ns, err) + continue + } + result[ns] = cs.IndexSizes + } + return result } const ( @@ -59,6 +200,13 @@ func main() { kong.Vars{"version": fmt.Sprintf("%s\nVersion %s\nBuild: %s using %s\nCommit: %s", toolname, Version, Build, GoVersion, Commit)}) + initLoggingForPTDEBUG() + + cmd := kongctx.Command() + if cmd == "" { + cmd = "(default)" + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -66,40 +214,121 @@ func main() { args.URI = "mongodb://" + args.URI } + log.Debugf("command=%s json=%v all-databases=%v all-collections=%v warmup-days=%.4f low-usage-threshold=%.4f large-index-size=%d include-low-usage=%v cross-reference-duplicates=%v", + cmd, args.JSON, args.AllDatabases, args.AllCollections, + args.WarmupDays, args.LowUsageThreshold, args.LargeIndexSize, + args.IncludeLowUsage, args.CrossReferenceDuplicates) + client, err := mongo.Connect(ctx, options.Client().ApplyURI(args.URI)) if err != nil { log.Fatalf("Cannot connect to the database: %q", err) } + if err := client.Ping(ctx, readpref.Primary()); err != nil { + log.Fatalf("Cannot connect to MongoDB at %s: %s", args.URI, err) + } + + log.Debugf("connected (redacted-uri=%s)", redactMongoURI(args.URI)) + if args.AllDatabases { - args.Databases, err = client.ListDatabaseNames(context.TODO(), primitive.D{}) + args.Databases, err = client.ListDatabaseNames(ctx, primitive.D{}) if err != nil { log.Fatalf("cannot list all databases: %s", err) } } + + if !args.AllDatabases && len(args.Databases) == 0 { + if dbName := extractDBFromURI(args.URI); dbName != "" { + args.Databases = []string{dbName} + } else { + log.Fatal("Error: specify --databases or --all-databases to select which databases to check") + } + } + if args.AllCollections { args.Collections = nil } + log.Debugf("databases (%d): %v explicit-collections=%v", + len(args.Databases), args.Databases, args.Collections) + + cfg := indexes.AnalysisConfig{ + WarmupDays: args.WarmupDays, + LowUsageThreshold: args.LowUsageThreshold, + LargeIndexSizeBytes: args.LargeIndexSize, + IncludeLowUsage: args.IncludeLowUsage, + CrossReferenceDuplicates: args.CrossReferenceDuplicates, + Now: time.Now(), + } + resp := response{} + var allAnalysis []indexes.IndexAnalysis + var duplicates []indexes.Duplicate + var dbCount, collCount int switch kongctx.Command() { case "check-unused": - resp.Unused = findUnused(ctx, client, args.Databases, args.Collections) + allAnalysis, dbCount, collCount = analyzeUnused(ctx, client, args.Databases, args.Collections, cfg, nil) + resp.Analysis = allAnalysis case "check-duplicates": - resp.Duplicated = findDuplicated(ctx, client, args.Databases, args.Collections) + duplicates = findDuplicated(ctx, client, args.Databases, args.Collections) + resp.Duplicated = duplicates case "check-all": - resp.Unused = findUnused(ctx, client, args.Databases, args.Collections) - resp.Duplicated = findDuplicated(ctx, client, args.Databases, args.Collections) + duplicates = findDuplicated(ctx, client, args.Databases, args.Collections) + resp.Duplicated = duplicates + allAnalysis, dbCount, collCount = analyzeUnused(ctx, client, args.Databases, args.Collections, cfg, duplicates) + resp.Analysis = allAnalysis default: kong.DefaultHelpPrinter(kong.HelpOptions{}, kongctx) + return } - fmt.Println(output(resp, args.JSON)) + showDupReport := kongctx.Command() == "check-duplicates" || kongctx.Command() == "check-all" + var dupReport duplicateReportData + if showDupReport { + var sizeByNS map[string]map[string]int64 + if len(duplicates) > 0 { + sizeByNS = collectDuplicateSizes(ctx, client, duplicates) + } + dupReport = buildDuplicateReportData(duplicates, sizeByNS) + } + + fmt.Println(output(resp, args.JSON, allAnalysis, dbCount, collCount, dupReport, showDupReport)) +} + +// ptdebugEnabled mirrors Perl toolkit truthiness: $ENV{PTDEBUG} || 0 (so "0", unset, and "" are off). +func ptdebugEnabled() bool { + v := os.Getenv("PTDEBUG") + return v != "" && v != "0" +} + +func initLoggingForPTDEBUG() { + log.SetOutput(os.Stderr) + if ptdebugEnabled() { + log.SetLevel(log.DebugLevel) + log.SetFormatter(&log.TextFormatter{FullTimestamp: true}) + return + } + log.SetLevel(log.WarnLevel) +} + +// redactMongoURI returns a copy of uri with user password removed for safe logging. +func redactMongoURI(uri string) string { + parsed, err := url.Parse(uri) + if err != nil { + return "" + } + if parsed.User != nil { + if _, hasPass := parsed.User.Password(); hasPass { + username := parsed.User.Username() + parsed.User = url.UserPassword(username, "xxx") + } + } + return parsed.String() } -func output(resp response, asJson bool) string { - if asJson { +func output(resp response, asJSON bool, analysis []indexes.IndexAnalysis, dbCount, collCount int, dupReport duplicateReportData, showDupReport bool) string { + if asJSON { jsonStr, err := json.MarshalIndent(resp, "", "\t") if err != nil { log.Fatal("cannot encode the response as json") @@ -109,47 +338,201 @@ func output(resp response, asJson bool) string { buf := new(bytes.Buffer) - t := template.Must(template.New("duplicated").Parse(templates.Duplicated)) - if err := t.Execute(buf, resp.Duplicated); err != nil { - log.Fatal(errors.Wrap(err, "cannot parse clusterwide section of the output template")) + if showDupReport { + renderDuplicateReport(buf, dupReport) } - t = template.Must(template.New("unused").Parse(templates.Unused)) - if err := t.Execute(buf, resp.Unused); err != nil { - log.Fatal(errors.Wrap(err, "cannot parse clusterwide section of the output template")) + if len(analysis) > 0 { + renderAnalysisReport(buf, analysis, dbCount, collCount) + } else { + t := template.Must(template.New("unused").Parse(templates.Unused)) + if err := t.Execute(buf, resp.Unused); err != nil { + log.Fatal(errors.Wrap(err, "cannot render unused indexes template")) + } } return buf.String() } -func findUnused(ctx context.Context, client *mongo.Client, databases []string, collections []string) []indexes.IndexStat { - unused := []indexes.IndexStat{} - var err error +func renderDuplicateReport(buf *bytes.Buffer, data duplicateReportData) { + funcMap := template.FuncMap{ + "formatBytes": indexes.FormatBytes, + "collName": func(ns string) string { + parts := strings.SplitN(ns, ".", 2) + if len(parts) == 2 { + return parts[1] + } + return ns + }, + } + t := template.Must(template.New("duplicated").Funcs(funcMap).Parse(templates.Duplicated)) + if err := t.Execute(buf, data); err != nil { + log.Fatal(errors.Wrap(err, "cannot render duplicated indexes template")) + } +} + +func renderAnalysisReport(buf *bytes.Buffer, analysis []indexes.IndexAnalysis, dbCount, collCount int) { + data := analysisReportData{ + ObsEnd: time.Now().UTC().Format(time.RFC3339), + TotalAnalyzed: len(analysis), + DatabaseCount: dbCount, + CollectionCount: collCount, + } + + var oldestSince time.Time + var writeRate float64 + + for _, a := range analysis { + if oldestSince.IsZero() || (!a.AccessSince.IsZero() && a.AccessSince.Before(oldestSince)) { + oldestSince = a.AccessSince + } + if a.WriteOpsPerSec > writeRate { + writeRate = a.WriteOpsPerSec + } + + switch a.Recommendation { + case indexes.RecommendSafeToDrop: + data.SafeToDrop = append(data.SafeToDrop, a) + data.SafeToDropSavings += a.IndexSizeBytes + case indexes.RecommendLikelyUnused: + data.LikelyUnused = append(data.LikelyUnused, a) + case indexes.RecommendLowUsage: + data.LowUsage = append(data.LowUsage, a) + case indexes.RecommendMonitor: + data.Monitor = append(data.Monitor, a) + case indexes.RecommendKeepConstraint, indexes.RecommendKeepHidden, indexes.RecommendKeepPartial: + data.Keep = append(data.Keep, a) + } + } + + if !oldestSince.IsZero() { + data.ObsStart = oldestSince.UTC().Format(time.RFC3339) + data.ObsDays = fmt.Sprintf("%.1f", time.Since(oldestSince).Hours()/24) + } else { + data.ObsStart = "unknown" + data.ObsDays = "N/A" + } + data.WriteRate = fmt.Sprintf("%.0f", writeRate) + data.SafeToDropCount = len(data.SafeToDrop) + data.LikelyUnusedCount = len(data.LikelyUnused) + data.LowUsageCount = len(data.LowUsage) + data.MonitorCount = len(data.Monitor) + data.KeepCount = len(data.Keep) + + funcMap := template.FuncMap{ + "formatBytes": indexes.FormatBytes, + "collName": func(ns string) string { + parts := strings.SplitN(ns, ".", 2) + if len(parts) == 2 { + return parts[1] + } + return ns + }, + "tagFor": func(a indexes.IndexAnalysis) string { + switch { + case a.IsUnique: + return " [UNIQUE]" + case a.IsTTL: + return " [TTL]" + case a.IsHidden: + return " [HIDDEN]" + default: + return "" + } + }, + } + + t := template.Must(template.New("analysis").Funcs(funcMap).Parse(templates.Analysis)) + if err := t.Execute(buf, data); err != nil { + log.Fatal(errors.Wrap(err, "cannot render analysis template")) + } +} + +func analyzeUnused( + ctx context.Context, + client *mongo.Client, + databases, collections []string, + cfg indexes.AnalysisConfig, + duplicates []indexes.Duplicate, +) ([]indexes.IndexAnalysis, int, int) { + var allAnalysis []indexes.IndexAnalysis + var dbCount, collCount int + + wr, err := indexes.CollectServerWriteRate(ctx, client) + if err != nil { + log.Warnf("cannot get server write rate (will use 0): %s", err) + } else if log.IsLevelEnabled(log.DebugLevel) { + log.Debugf("server write-rate: %.6f ops/s uptime-sec=%d", wr.WriteOpsPerSec, wr.Uptime) + } colls := make([]string, len(collections)) copy(colls, collections) for _, database := range databases { + if indexes.IsSystemDB(database) { + continue + } + dbCount++ + if len(collections) == 0 { colls, err = client.Database(database).ListCollectionNames(ctx, primitive.D{}) if err != nil { - log.Errorf("cannot get the list of collections for the database %s", database) + log.Errorf("cannot get the list of collections for the database %s: %s", database, err) continue } } for _, collection := range colls { - idx, err := indexes.FindUnused(ctx, client, database, collection) + if indexes.IsSystemCollection(collection) { + continue + } + + collCount++ + ns := database + "." + collection + + log.Debugf("analyze-unused start namespace=%s", ns) + + stats, err := indexes.CollectIndexStats(ctx, client, database, collection) + if err != nil { + log.Errorf("error collecting $indexStats for %s: %s", ns, err) + continue + } + stats = indexes.AggregateShardStats(stats) + + metadata, err := indexes.CollectIndexMetadata(ctx, client, database, collection) if err != nil { - log.Errorf("error while checking unused indexes in %s.%s: %s", database, collection, err) + log.Errorf("error collecting index metadata for %s: %s", ns, err) continue } - unused = append(unused, idx...) + cs, err := indexes.CollectCollStats(ctx, client, database, collection) + if err != nil { + log.Warnf("cannot get collStats for %s (sizes will be 0): %s", ns, err) + } + + records := indexes.DeduplicateIndexRecords(indexes.BuildIndexRecords(stats, metadata, cs, wr, ns)) + + log.Debugf("analyze-unused done namespace=%s indexStats=%d metadata=%d records=%d", + ns, len(stats), len(metadata), len(records)) + + if cfg.CrossReferenceDuplicates && duplicates != nil { + indexes.CrossReferenceDuplicates(records, duplicates, stats) + } + + for _, rec := range records { + a := indexes.ScoreIndex(rec, cfg) + if a.Recommendation == "" { + continue + } + if a.Recommendation == indexes.RecommendLowUsage && !cfg.IncludeLowUsage { + continue + } + allAnalysis = append(allAnalysis, a) + } } } - return unused + return allAnalysis, dbCount, collCount } func findDuplicated(ctx context.Context, client *mongo.Client, databases []string, collections []string) []indexes.Duplicate { @@ -160,6 +543,10 @@ func findDuplicated(ctx context.Context, client *mongo.Client, databases []strin copy(colls, collections) for _, database := range databases { + if indexes.IsSystemDB(database) { + continue + } + if len(collections) == 0 { colls, err = client.Database(database).ListCollectionNames(ctx, primitive.D{}) if err != nil { @@ -169,15 +556,41 @@ func findDuplicated(ctx context.Context, client *mongo.Client, databases []strin } for _, collection := range colls { + if indexes.IsSystemCollection(collection) { + continue + } + + log.Debugf("check-duplicates start namespace=%s.%s", database, collection) + dups, err := indexes.FindDuplicated(ctx, client, database, collection) if err != nil { log.Errorf("error while checking duplicated indexes in %s.%s: %s", database, collection, err) continue } + log.Debugf("check-duplicates done namespace=%s.%s duplicate-groups=%d", database, collection, len(dups)) + duplicated = append(duplicated, dups...) } } return duplicated } + +// extractDBFromURI parses the MongoDB connection URI and returns the database +// name if one is present in the path component (e.g., mongodb://host:port/mydb). +// Returns empty string if no database is specified or the URI is the "admin" default. +func extractDBFromURI(uri string) string { + parsed, err := url.Parse(uri) + if err != nil { + return "" + } + if parsed.Scheme != "mongodb" && parsed.Scheme != "mongodb+srv" { + return "" + } + db := strings.TrimPrefix(parsed.Path, "/") + if db == "" || db == "admin" { + return "" + } + return db +} diff --git a/src/go/pt-mongodb-index-check/main_test.go b/src/go/pt-mongodb-index-check/main_test.go index d4fcf1ec3..4c2158f76 100644 --- a/src/go/pt-mongodb-index-check/main_test.go +++ b/src/go/pt-mongodb-index-check/main_test.go @@ -1,9 +1,16 @@ package main import ( + "bytes" "os/exec" "regexp" + "strings" "testing" + + "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/indexes" ) /* @@ -20,3 +27,268 @@ func TestVersionOption(t *testing.T) { t.Errorf("%s --version returns wrong result:\n%s", toolname, out) } } + +func TestExtractDBFromURI(t *testing.T) { + tests := []struct { + uri string + want string + }{ + {"mongodb://localhost:27017/mydb", "mydb"}, + {"mongodb://localhost:27017/admin", ""}, + {"mongodb://localhost:27017", ""}, + {"mongodb://localhost:27017/", ""}, + {"mongodb://user:pass@host:27017/testdb?authSource=admin", "testdb"}, + {"mongodb+srv://host/appdb", "appdb"}, + {"not-a-valid-uri", ""}, + } + + for _, tt := range tests { + t.Run(tt.uri, func(t *testing.T) { + got := extractDBFromURI(tt.uri) + if got != tt.want { + t.Errorf("extractDBFromURI(%q) = %q, want %q", tt.uri, got, tt.want) + } + }) + } +} + +func TestRedactMongoURI(t *testing.T) { + tests := []struct { + in string + want string // substring checks unless full is set via containsOnly + }{ + { + in: "mongodb://user:secret@host:27017/mydb?authSource=admin", + want: "mongodb://user:xxx@host:27017/mydb?authSource=admin", + }, + { + in: "mongodb://host:27017/mydb", + want: "mongodb://host:27017/mydb", + }, + { + in: "mongodb+srv://u:p%40ssword@cluster.example/dbname", + want: "mongodb+srv://u:xxx@cluster.example/dbname", + }, + { + in: "mongodb://onlyuser@host:27017/", + want: "mongodb://onlyuser@host:27017/", + }, + } + + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + got := redactMongoURI(tt.in) + if got != tt.want { + t.Errorf("redactMongoURI(%q) = %q, want %q", tt.in, got, tt.want) + } + if strings.Contains(got, "secret") || strings.Contains(got, "p%40ssword") { + t.Errorf("redactMongoURI leaked credential in %q", got) + } + }) + } +} + +func TestRedactMongoURI_invalid(t *testing.T) { + if g := redactMongoURI("://"); g != "" { + t.Errorf("expected unparseable sentinel, got %q", g) + } +} + +func TestBuildDuplicateReportData(t *testing.T) { + t.Run("empty", func(t *testing.T) { + d := buildDuplicateReportData(nil, nil) + assert.Equal(t, 0, d.TotalPairs) + assert.Empty(t, d.Standard) + assert.Empty(t, d.WithWarning) + assert.Equal(t, 0, d.DatabaseCount) + assert.Equal(t, 0, d.CollectionCount) + }) + + t.Run("partition standard vs warning and action", func(t *testing.T) { + dups := []indexes.Duplicate{ + { + Namespace: "mydb.orders", + Name: "idx_a", + Key: indexes.IndexKey{{Key: "a", Value: int32(1)}}, + ContainerName: "idx_ab", + ContainerKey: indexes.IndexKey{ + {Key: "a", Value: int32(1)}, + {Key: "b", Value: int32(1)}, + }, + }, + { + Namespace: "mydb.users", + Name: "email_1", + Key: indexes.IndexKey{{Key: "email", Value: int32(1)}}, + ContainerName: "email_status", + ContainerKey: indexes.IndexKey{ + {Key: "email", Value: int32(1)}, + {Key: "status", Value: int32(1)}, + }, + Warning: "prefix index enforces unique constraint", + }, + } + sizes := map[string]map[string]int64{ + "mydb.orders": {"idx_a": 100, "idx_ab": 500}, + "mydb.users": {"email_1": 50, "email_status": 200}, + } + d := buildDuplicateReportData(dups, sizes) + assert.Equal(t, 2, d.TotalPairs) + assert.Equal(t, 1, d.DatabaseCount, "distinct database prefix") + assert.Equal(t, 2, d.CollectionCount, "distinct namespaces") + assert.Len(t, d.Standard, 1) + assert.Len(t, d.WithWarning, 1) + assert.Contains(t, d.Standard[0].Action, `dropIndex("idx_a")`) + assert.Contains(t, d.Standard[0].Action, "db.orders.") + assert.True(t, d.Standard[0].HasSizes) + assert.Equal(t, int64(100), d.Standard[0].PrefixSize) + assert.Equal(t, int64(500), d.Standard[0].ContainerSize) + assert.Contains(t, d.Standard[0].Reason, "idx_a") + assert.Contains(t, d.Standard[0].Reason, "idx_ab") + assert.Equal(t, "prefix index enforces unique constraint", d.WithWarning[0].Warning) + assert.Contains(t, d.WithWarning[0].Action, `dropIndex("email_1")`) + assert.Contains(t, d.WithWarning[0].Action, "db.users.") + }) + + t.Run("nil size map omits sizes", func(t *testing.T) { + d := buildDuplicateReportData([]indexes.Duplicate{ + { + Namespace: "app.coll", + Name: "x_1", + Key: indexes.IndexKey{{Key: "x", Value: int32(1)}}, + ContainerName: "x_y", + ContainerKey: indexes.IndexKey{{Key: "x", Value: int32(1)}, {Key: "y", Value: int32(-1)}}, + }, + }, nil) + assert.Len(t, d.Standard, 1) + assert.False(t, d.Standard[0].HasSizes) + }) +} + +func TestRenderDuplicateReport_containsSections(t *testing.T) { + data := buildDuplicateReportData([]indexes.Duplicate{ + { + Namespace: "shop.orders", + Name: "region_1", + Key: indexes.IndexKey{{Key: "region", Value: int32(1)}}, + ContainerName: "region_created", + ContainerKey: indexes.IndexKey{ + {Key: "region", Value: int32(1)}, + {Key: "created", Value: int32(-1)}, + }, + }, + }, nil) + + buf := new(bytes.Buffer) + renderDuplicateReport(buf, data) + out := buf.String() + assert.Contains(t, out, "Duplicate Prefix Index Report") + assert.Contains(t, out, "REDUNDANT PREFIX") + assert.Contains(t, out, "shop.orders") + assert.Contains(t, out, "dropIndex(\"region_1\")") + assert.Contains(t, out, "Summary:") +} + +// Regression: unique prefix warning must appear in rendered duplicate report +func TestRenderDuplicateReport_uniqueWarning(t *testing.T) { + data := buildDuplicateReportData([]indexes.Duplicate{ + { + Namespace: "mydb.users", + Name: "email_unique", + Key: indexes.IndexKey{{Key: "email", Value: int32(1)}}, + ContainerName: "email_status", + ContainerKey: indexes.IndexKey{ + {Key: "email", Value: int32(1)}, + {Key: "status", Value: int32(1)}, + }, + Warning: "prefix index enforces unique constraint; dropping requires the container index to also be unique", + }, + }, nil) + + buf := new(bytes.Buffer) + renderDuplicateReport(buf, data) + out := buf.String() + assert.Contains(t, out, "UNIQUE / CONSTRAINT WARNING") + assert.Contains(t, out, "[UNIQUE]") + assert.Contains(t, out, "WARNING: prefix index enforces unique constraint") + assert.Contains(t, out, `dropIndex("email_unique")`) + assert.Empty(t, data.Standard, "unique pair should not be in Standard section") + assert.Len(t, data.WithWarning, 1) +} + +// Regression: check-unused output must NOT include the duplicate report section +func TestOutput_checkUnused_noDuplicateSection(t *testing.T) { + resp := response{} + analysis := []indexes.IndexAnalysis{ + { + Namespace: "mydb.orders", + IndexName: "idx_old", + IndexKey: primitive.D{{Key: "old", Value: int32(1)}}, + Recommendation: indexes.RecommendMonitor, + Reason: "test reason", + }, + } + out := output(resp, false, analysis, 1, 1, duplicateReportData{}, false) + assert.NotContains(t, out, "Duplicate Prefix Index Report") + assert.Contains(t, out, "Unused Index Analysis") +} + +// Regression: check-all output must include BOTH duplicate and unused sections +func TestOutput_checkAll_bothSections(t *testing.T) { + dups := []indexes.Duplicate{ + { + Namespace: "mydb.orders", + Name: "idx_a", + Key: indexes.IndexKey{{Key: "a", Value: int32(1)}}, + ContainerName: "idx_ab", + ContainerKey: indexes.IndexKey{ + {Key: "a", Value: int32(1)}, + {Key: "b", Value: int32(1)}, + }, + }, + } + dupReport := buildDuplicateReportData(dups, nil) + resp := response{Duplicated: dups} + analysis := []indexes.IndexAnalysis{ + { + Namespace: "mydb.orders", + IndexName: "idx_test", + IndexKey: primitive.D{{Key: "test", Value: int32(1)}}, + Recommendation: indexes.RecommendMonitor, + Reason: "monitoring", + }, + } + out := output(resp, false, analysis, 1, 1, dupReport, true) + assert.Contains(t, out, "Duplicate Prefix Index Report") + assert.Contains(t, out, "Unused Index Analysis") + assert.Contains(t, out, `dropIndex("idx_a")`) +} + +// Regression: no database selected must produce an error, not blank output +func TestExtractDBFromURI_adminReturnsEmpty(t *testing.T) { + assert.Equal(t, "", extractDBFromURI("mongodb://localhost:27017/admin")) + assert.Equal(t, "", extractDBFromURI("mongodb://localhost:27017")) + assert.Equal(t, "", extractDBFromURI("mongodb://localhost:27017/")) + assert.Equal(t, "", extractDBFromURI("not-a-uri")) +} + +func TestPtdebugEnabled(t *testing.T) { + t.Run("off when unset", func(t *testing.T) { + t.Setenv("PTDEBUG", "") + if ptdebugEnabled() { + t.Fatal("expected false when PTDEBUG empty") + } + }) + t.Run("off for zero", func(t *testing.T) { + t.Setenv("PTDEBUG", "0") + if ptdebugEnabled() { + t.Fatal("expected false when PTDEBUG=0") + } + }) + t.Run("on for one", func(t *testing.T) { + t.Setenv("PTDEBUG", "1") + if !ptdebugEnabled() { + t.Fatal("expected true when PTDEBUG=1") + } + }) +} diff --git a/src/go/pt-mongodb-index-check/templates/analysis.go b/src/go/pt-mongodb-index-check/templates/analysis.go new file mode 100644 index 000000000..2bb7c63c2 --- /dev/null +++ b/src/go/pt-mongodb-index-check/templates/analysis.go @@ -0,0 +1,42 @@ +package templates + +var Analysis = ` +# ============================================================ +# Unused Index Analysis +# ============================================================ +# Observation window: {{ .ObsStart }} to {{ .ObsEnd }} ({{ .ObsDays }} days) +# Indexes analyzed: {{ .TotalAnalyzed }} across {{ .DatabaseCount }} database(s), {{ .CollectionCount }} collection(s) +# Server write rate: ~{{ .WriteRate }} ops/sec +{{ if .SafeToDrop }} +# ---- SAFE TO DROP (high confidence) -------------------------- +{{ range .SafeToDrop }} + {{ .Namespace }} index '{{ .IndexName }}' { {{- range $i, $val := .IndexKey }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} } + Ops: {{ .AccessOps }} in {{ .AgeDays }} days | Size: {{ formatBytes .IndexSizeBytes }} | Score: {{ printf "%.2f" .Score }} + Reason: {{ .Reason }} + Action: db.{{ collName .Namespace }}.dropIndex("{{ .IndexName }}") +{{ end }}{{ end }}{{ if .LikelyUnused }} +# ---- LIKELY UNUSED (review recommended) ---------------------- +{{ range .LikelyUnused }} + {{ .Namespace }} index '{{ .IndexName }}' { {{- range $i, $val := .IndexKey }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} } + Ops: {{ .AccessOps }} in {{ .AgeDays }} days | Size: {{ formatBytes .IndexSizeBytes }} | Score: {{ printf "%.2f" .Score }} + Reason: {{ .Reason }} +{{ end }}{{ end }}{{ if .LowUsage }} +# ---- LOW USAGE (non-zero but minimal) ------------------------ +{{ range .LowUsage }} + {{ .Namespace }} index '{{ .IndexName }}' { {{- range $i, $val := .IndexKey }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} } + Ops: {{ .AccessOps }} in {{ .AgeDays }} days ({{ printf "%.1f" .OpsPerDay }}/day) | Size: {{ formatBytes .IndexSizeBytes }} | Score: {{ printf "%.2f" .Score }} + Reason: {{ .Reason }} +{{ end }}{{ end }}{{ if .Monitor }} +# ---- MONITOR (insufficient data) ----------------------------- +{{ range .Monitor }} + {{ .Namespace }} index '{{ .IndexName }}' { {{- range $i, $val := .IndexKey }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} } + Ops: {{ .AccessOps }} in {{ .AgeDays }} days | Size: {{ formatBytes .IndexSizeBytes }} | Score: {{ printf "%.2f" .Score }} + Reason: {{ .Reason }} +{{ end }}{{ end }}{{ if .Keep }} +# ---- KEEP (constraints / special) ---------------------------- +{{ range .Keep }} + {{ .Namespace }} index '{{ .IndexName }}' { {{- range $i, $val := .IndexKey }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} }{{ tagFor . }} + Ops: {{ .AccessOps }} in {{ .AgeDays }} days | Kept: {{ .Reason }} +{{ end }}{{ end }} +# Summary: {{ .SafeToDropCount }} safe to drop{{ if gt .SafeToDropSavings 0 }} (saving ~{{ formatBytes .SafeToDropSavings }}){{ end }}, {{ .LikelyUnusedCount }} likely unused, {{ .LowUsageCount }} low usage, {{ .MonitorCount }} monitoring, {{ .KeepCount }} kept (constraints) +` diff --git a/src/go/pt-mongodb-index-check/templates/duplicated.go b/src/go/pt-mongodb-index-check/templates/duplicated.go index 048d3cb0b..39f942abf 100644 --- a/src/go/pt-mongodb-index-check/templates/duplicated.go +++ b/src/go/pt-mongodb-index-check/templates/duplicated.go @@ -1,10 +1,33 @@ package templates -// {{if $i}},{{end}} adds a comma after the first element. -// When $i == 0 (first element) {{ if $i }} returns false (0) +// Duplicated is the sectioned text template for the duplicate prefix index +// report. It is rendered by renderDuplicateReport in main.go which supplies +// a duplicateReportData value and registers formatBytes / collName helpers. var Duplicated = ` -Duplicated indexes -{{ range . }} -{{ .Namespace }}, index '{{ .Name }}', with fields { {{- range $i, $val := .Key }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} } is the prefix of '{{ .ContainerName }}' with fields { {{- range $i, $val := .ContainerKey }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} }{{ end}} +# ============================================================ +# Duplicate Prefix Index Report +# ============================================================ +# Pairs found: {{ .TotalPairs }} across {{ .DatabaseCount }} database(s), {{ .CollectionCount }} collection(s) +{{ if eq .TotalPairs 0 }} +# No duplicate prefix indexes detected. +{{ end }}{{ if .Standard }} +# ---- REDUNDANT PREFIX (shorter index is candidate to drop) --- +{{ range .Standard }} + {{ .Namespace }} + Prefix: '{{ .Name }}' { {{- range $i, $v := .Key }}{{if $i}}, {{end}}{{ $v.Key }}:{{ $v.Value }}{{ end -}} }{{ if .HasSizes }} {{ formatBytes .PrefixSize }}{{ end }} + Container: '{{ .ContainerName }}' { {{- range $i, $v := .ContainerKey }}{{if $i}}, {{end}}{{ $v.Key }}:{{ $v.Value }}{{ end -}} }{{ if .HasSizes }} {{ formatBytes .ContainerSize }}{{ end }} + Reason: {{ .Reason }} + Action: {{ .Action }} +{{ end }}{{ end }}{{ if .WithWarning }} +# ---- UNIQUE / CONSTRAINT WARNING ---------------------------- +{{ range .WithWarning }} + {{ .Namespace }} + Prefix: '{{ .Name }}' { {{- range $i, $v := .Key }}{{if $i}}, {{end}}{{ $v.Key }}:{{ $v.Value }}{{ end -}} } [UNIQUE]{{ if .HasSizes }} {{ formatBytes .PrefixSize }}{{ end }} + Container: '{{ .ContainerName }}' { {{- range $i, $v := .ContainerKey }}{{if $i}}, {{end}}{{ $v.Key }}:{{ $v.Value }}{{ end -}} }{{ if .HasSizes }} {{ formatBytes .ContainerSize }}{{ end }} + Reason: {{ .Reason }} + WARNING: {{ .Warning }} + Action: {{ .Action }} +{{ end }}{{ end }} +# Summary: {{ len .Standard }} redundant prefix pair(s), {{ len .WithWarning }} with unique/constraint warning(s) `