From 7e838bb8a2c071e7bfc2b208b7f8bba8516b99e7 Mon Sep 17 00:00:00 2001 From: sunby Date: Tue, 8 Sep 2026 20:41:25 +0800 Subject: [PATCH] enhance: add event-driven stats discovery Signed-off-by: sunby --- configs/milvus.yaml | 3 + ...60908-stats-inspector-event-driven-plan.md | 364 +++++++++++++ internal/datacoord/meta.go | 58 ++- .../stats_discovery_benchmark_test.go | 60 +++ .../stats_discovery_compaction_test.go | 72 +++ .../datacoord/stats_discovery_failure_test.go | 253 +++++++++ internal/datacoord/stats_discovery_meta.go | 76 +++ internal/datacoord/stats_inspector.go | 110 +++- internal/datacoord/stats_reconcile.go | 331 ++++++++++++ internal/datacoord/stats_reconcile_queue.go | 330 ++++++++++++ .../datacoord/stats_reconcile_queue_test.go | 165 ++++++ internal/datacoord/stats_reconcile_test.go | 493 ++++++++++++++++++ internal/datacoord/stats_task_meta.go | 6 + pkg/metrics/datacoord_metrics.go | 2 + pkg/metrics/stats_discovery_metrics.go | 57 ++ pkg/metrics/stats_discovery_metrics_test.go | 50 ++ pkg/util/paramtable/component_param.go | 11 +- pkg/util/paramtable/stats_discovery.go | 49 ++ pkg/util/paramtable/stats_discovery_test.go | 50 ++ pkg/util/typeutil/field_match_test.go | 66 +++ pkg/util/typeutil/field_schema.go | 16 + 21 files changed, 2572 insertions(+), 50 deletions(-) create mode 100644 docs/design-docs/design_docs/20260908-stats-inspector-event-driven-plan.md create mode 100644 internal/datacoord/stats_discovery_benchmark_test.go create mode 100644 internal/datacoord/stats_discovery_compaction_test.go create mode 100644 internal/datacoord/stats_discovery_failure_test.go create mode 100644 internal/datacoord/stats_discovery_meta.go create mode 100644 internal/datacoord/stats_reconcile.go create mode 100644 internal/datacoord/stats_reconcile_queue.go create mode 100644 internal/datacoord/stats_reconcile_queue_test.go create mode 100644 internal/datacoord/stats_reconcile_test.go create mode 100644 pkg/metrics/stats_discovery_metrics.go create mode 100644 pkg/metrics/stats_discovery_metrics_test.go create mode 100644 pkg/util/paramtable/stats_discovery.go create mode 100644 pkg/util/paramtable/stats_discovery_test.go create mode 100644 pkg/util/typeutil/field_match_test.go diff --git a/configs/milvus.yaml b/configs/milvus.yaml index fd6524a1002..d53aadfac0a 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -762,6 +762,9 @@ indexNode: dataCoord: taskCheckInterval: 1 + statsInspector: + discoveryMode: poll # poll: legacy discovery; shadow: read-only event checks; event: event-driven discovery. Restart required. + reconcileInterval: 600 # Seconds. Startup and lost-notification reconciliation use the same budget. channel: watchTimeoutInterval: 300 # Timeout on watching channels (in seconds). Datanode tickler update watch progress will reset timeout timer. legacyVersionWithoutRPCWatch: 2.4.1 # Datanodes <= this version are considered as legacy nodes, which doesn't have rpc based watch(). This is only used during rolling upgrade where legacy nodes won't get new channels diff --git a/docs/design-docs/design_docs/20260908-stats-inspector-event-driven-plan.md b/docs/design-docs/design_docs/20260908-stats-inspector-event-driven-plan.md new file mode 100644 index 00000000000..6a42dc8742d --- /dev/null +++ b/docs/design-docs/design_docs/20260908-stats-inspector-event-driven-plan.md @@ -0,0 +1,364 @@ +# StatsInspector 事件驱动改造计划 + +状态：已在隔离分支实现，默认仍为 poll；本地验证及限制见末尾实施记录。日期：2026-09-08。 + +以下第 1–10 节保留原计划的阶段要求；已实现内容、实测结果和未通过的验证见末尾“实施记录”，不将原计划中的验收目标视为已达成。 + +实施基于 `codex/load-1m-segments-pr-stack-rebased-qv-work`。初始实现与本地测试使用 `e99fe0e5e1f31fc8c84978fea98180b84a0b1f81`；提交前对齐远端 squash 后的 `7e32a5ca9b7e472f1563668e497595a1f57b0d7d`，两者的 DataCoord 源码一致。本次不包含部署或线上性能收益声明。 + +## 1. 结论和范围 + +建议先把 **StatsInspector 的任务发现改为事件驱动，保留低频、有界的补偿扫描**；IndexInspector 已有事件机制，作为后续独立改动补齐覆盖、去掉溢出时的同步全量扫描。 + +这里的事件驱动是：元数据发生相关变化时，记录“哪个 segment / collection 需要重新检查”；后台读取最新元数据，决定是否需要创建任务。事件只负责唤醒，不携带完整元数据，也不直接代表一条必须执行的任务。 + +第一阶段只处理 TextIndexJob 和 JsonKeyIndexJob 的发现、提交及补偿，不改变它们的业务资格条件、持久化任务格式和 worker 执行协议。 + +不在本次范围内： + +- 将所有 inspector、CompactionTriggerManager、TTL/超时检查和任务清理统一改成事件驱动。时间到期本身仍需定时器。 +- 改动 Load/Search、QueryView listener 或 `discoverableShards` 的语义。 +- 新增 WAL 消息、持久化事件队列、逐 segment 的 etcd watch 或线上全量 etcd 扫描。 +- 改动 DDL broadcast 的 ACK、回放、提交和重试语义。 +- 改动 QuotaCenter、PChannel affinity、Proxy SetRates，以及 SegmentInfo 冷热分层或 tombstone 清理。 +- 调整线上 GOGC、配置、实例规模或进行部署；本文只是修改计划。 + +对于“每次请求 load 新 shard 再 search”的模式，本改造仍可能减少后台固定开销，但 **Load/Search 本身不等于 stats 构建资格发生变化**。应监听 flush、sort/compaction、schema、manifest 等元数据变化，不能把每次 load 变成一次 stats 扫描。 + +## 2. 为什么优先改 StatsInspector + +### 2.1 可复现的分配路径 + +旧发现循环按周期遍历 collection/schema/segment，即使没有相关元数据变化，也会创建 collection/candidate slice 并重复解析字段参数。旧字段 helper 为参数创建 map，且缺失 `enable_match` 时构造错误；这些都是本次可以直接从代码和微基准验证的分配来源。 + +本 PR 只公开可复现的本地 benchmark 和验证边界，不附带运行实例标识、运维采样数据或本机 pprof 文件。具体分配结果见末尾实施记录；字段 helper 去分配与事件发现的收益有重叠，需分别衡量。 + +分配字节下降不能直接换算成 GC CPU 或 search p99 降幅。线上验收仍需分别观察 allocation assist、dedicated/fractional mark 和 idle mark，并使用同负载 A/B。 + +### 2.2 当前代码行为 + +| 位置 | 已核实的现状 | 对计划的影响 | +|---|---|---| +| [stats_inspector.go](../../../internal/datacoord/stats_inspector.go) 的 `triggerStatsTaskLoop` | 按 `TaskCheckInterval` 周期调用任务发现，Text/JSON 轮换先后顺序 | 替换正常发现方式，保留两类任务的公平性 | +| 同文件的 Text/JSON trigger | 分别 `GetCollections()`，检查 schema，再按 collection `SelectSegments()` | 不应只把 ticker 换成事件后继续扫全库 | +| 同文件的 `SubmitStatsTask` | 已存在任务、外部 collection 不满足条件、提交限流等分支都可能返回 `nil` | `nil` 不能作为“事件已完成”的唯一判断 | +| [stats_task_meta.go](../../../internal/datacoord/stats_task_meta.go) | `HasStatsTask` 包含 Finished/Failed，直到持久化任务清理后才释放二级索引 | 必须覆盖任务清理后重新检查的触发点 | +| [index_inspector.go](../../../internal/datacoord/index_inspector.go) | 已有 segment 事件、collection 事件和 pending 去重；全量补偿周期为 `TaskCheckInterval × 10` | 不是从零重写；后续处理补偿成本与覆盖缺口 | +| [go_channel_singleton.go](../../../internal/datacoord/go_channel_singleton.go) | segment index 通知溢出会唤醒一次全量补偿 | 新 Stats 队列不能照搬“溢出立即全扫” | +| [compaction_trigger_v2.go](../../../internal/datacoord/compaction_trigger_v2.go) | 消费现有 `statsTaskCh`，触发的是 sort compaction | 不复用该 channel 作为 StatsInspector 的 Text/JSON 队列 | + +当前参数定义中 `dataCoord.taskCheckInterval` 默认 60 秒，`dataCoord.statsTaskPendingLimit` 默认 100；这是代码默认值，不是本次线上有效值的声明。不要通过增大共享 `TaskCheckInterval` 一并改变其他调度循环。 + +## 3. 目标和必须保持的不变量 + +目标：稳定、无相关元数据变化时，不再高频反复遍历全部 collection/schema/segment；正常发现成本主要随实际变化量增长。全量补偿仍有总量成本，但摊到受控的低频扫描中。 + +必须保持： + +1. 当前 Text/JSON 资格条件不变，包括 Flushing/Flushed、非 L0、排序/namespace 排序、external 例外、字段配置、JSON 格式版本，以及 external JSON 的 Storage V3/manifest 条件。 +2. `common.enabledJSONShredding` 和旧 `dataCoord.jsonShreddingTriggerCount=0` 的禁用语义不变；BM25 当前未启用，不顺带开启。 +3. 先成功持久化并发布元数据，再发本地通知；通知失败或进程退出不回滚已经提交的业务操作。 +4. 事件允许重复、合并、乱序和进程内丢失；最终正确性来自最新元数据、已有持久化任务和补偿检查。 +5. 同一个 `(segmentID, subJobType)` 不因并发发现产生重复有效任务；不得把队列去重误当成持久化层的唯一性保证。 +6. 提交准入仍按 Stats 类型计数，包含现有 scheduler 的 backoff 任务；不扩大预算、不改变现有阈值比较语义。 +7. worker 执行失败由现有 GlobalScheduler 重试；发现阶段的暂缓由新的 pending 机制处理，两者不重复创建任务。 +8. 队列、重试状态、collection dirty 集合、schema 缓存都必须有上限；不能用无界 ID set 代替原来的短命 slice。 +9. 保留现有元数据 revision/CAS/tombstone 保护；本地事件序号不是持久化 revision，更不是 WAL TimeTick。 + +## 4. 事件从哪里来 + +### 4.1 接入边界 + +优先在 DataCoord **成功发布元数据后的公共出口** 接入实例级通知接口，业务入口用于核对覆盖，不逐处复制完整资格判断。 + +- segment：审计 `meta.UpdateSegmentsInfo` 及其他直接提交并更新 segment cache 的路径。事务成功、该批 cache 发布完成后，通知可能影响资格的 segment ID。 +- collection：在 DataCoord schema/cache 成功更新后标记 collection dirty。只读取最新 schema；不让 RootCoord 等待 stats 任务完成。 +- task：持久化任务清理且二级索引移除后，通知对应 segment/subjob 重新检查。 +- config：使用现有 ParamTable `Watch/Unwatch`，回调只更新本地规则 epoch 并标记补偿；不在配置回调中扫描全库。 + +不在 CAS 重试用的 mutation closure 内发事件，也不在通用 `Cache.Insert` 中嵌入 Stats 业务。恢复、过期 revision、批量发布和业务提交的语义不同，不能因底层 cache 写入就假定“新任务已经具备条件”。 + +通知只需固定大小的 ID、原因枚举/位图、可选版本。不要 clone SegmentInfo、schema、binlog 或 manifest 内容进队列。多次相同状态更新合并；不相关的 heartbeat/统计数更新不应制造反复检查。 + +### 4.2 生产者审计清单 + +下表列出已定位的代码入口和实施时必须补齐的链路；**不是声称所有资格修改点已完成端到端审计**。关闭高频旧扫描前，必须逐项追踪到真实持久化和 cache 发布点。 + +| 变化 | 已定位的入口 | 计划动作及核查点 | +|---|---|---| +| Flush 完成 | `server.go` 的 `flushFlushingSegment`、`postFlush` | 从真实状态提交处触发检查；不能仅依赖会分流到 sort compaction 的旧通知 | +| Sort/mix compaction 完成 | `compaction_task_mix.go`、`task_stats.go` | 新 segment 元数据可读后通知；同时覆盖旧 sort stats 路径和现行 sort compaction 路径 | +| Clustering/schema bump | `compaction_task_clustering.go`、`compaction_task_bump_schema_version.go` | 覆盖临时 segment、最终结果以及原 ID 原地更新；保持可见性和排序条件 | +| Import | `import_checker.go`、`import_util.go`、`ddl_callbacks_import.go` | 审计 segment 创建、排序完成、import commit/abort；不要凭通知名称猜测 eligibility，也不改变两阶段提交可见性 | +| Manifest/外部数据刷新 | `ddl_callbacks_batch_update_manifest.go`、`task_refresh_external_collection.go` | 在实际元数据更新后通知；重复 manifest 不应不断创建任务；新增、更新、删除均需覆盖 | +| Schema/collection cache 变化 | `services.go:BroadcastAlteredCollection`、`meta.go:AddCollection`、`server.go:loadCollectionFromRootCoord` | 新 schema 可见后标记 collection；RootCoord 先推 schema 再发布绑定 index 的既有顺序保持不变 | +| Schema bump 的延迟完成 | `meta.go` 的 bump-schema compaction mutation | schema 先更新但 segment 尚不满足条件时，后续 segment 变化必须再次唤醒 | +| 任务结束与清理 | `task_stats.go`、`stats_task_meta.go:DropStatsTask` | 结果写入后检查后续需求；Finished/Failed 仍占二级索引时不能新建，清理成功后再唤醒 | +| 功能开关、格式升级 | `pkg/util/paramtable/component_param.go`、`common.JSONStatsDataFormatVersion` | 热更新启用时启动分批补偿；编译期格式升级由启动补偿覆盖 | +| Drop/truncate/restore | `meta.go`、`handler.go:FinishDropChannel`、`snapshot_manager.go` 及相关 DDL 路径 | 删除后旧事件不复活对象；snapshot restore 的 `AddSegment` 路径不能遗漏；collection 缓存缺失不能一律当成永久删除 | +| 启动和 leader 切换 | `server.go`、`statsInspector.Start/reloadFromMeta` | 恢复已有任务并进行一次有界发现，修补上个进程未送达的通知 | + +源头审计至少搜索：`SetSegment`、`AddSegment`、`UpdateSegmentsInfo`、状态/排序/schema/manifest/stats 更新、`AddCollection/DropCollection`、任务二级索引增删，而不只搜索旧的 `notifySegmentIndexBuild`。 + +## 5. 消费、去重、重试设计 + +### 5.1 实例级队列 + +新增 Stats 专用的 pending 管理器，由 DataCoord Server/StatsInspector 的生命周期持有；第一版不抽象为全系统 EventBus，不新增全局 singleton。 + +工作分两类： + +- segment 工作：key 为 `(segmentID, subJobType)`；只保存 generation、最早 dirty 时间、原因、重试时间等轻量状态。 +- collection 工作：保存 collection ID 和扫描进度，用于 schema 改动；按批枚举 segment，不能一次展开成全量 segment 事件。 + +待处理、处理中和延迟重试的 key 共享总容量预算，避免出队后重试容器无限增长。同 key 一次只允许一个消费者执行；大 collection 不能一次占满全部执行预算。Text 和 JSON 使用轮转/配额，至少保留现有交替机制的无饥饿性质。 + +唤醒 channel 可以只有一个合并信号：先记录 dirty，再尝试非阻塞唤醒。信号不是事件存储本身，合并唤醒不等于丢掉 dirty 状态。 + +### 5.2 防止“处理期间来的新事件被清掉” + +处理 key 时记录 generation；执行完成后，只能确认这一代的检查。若处理中 generation 又增加，则保留 pending 再检查。不得 `pop` 后无条件删除同 key 的新状态。 + +所有资格判断读取当前元数据。旧事件可以触发多一次无害检查，但不能把旧 schema、旧 manifest 或已删除 segment 写回 cache。新旧 Inspector 切换时，旧实例的回调必须失效。 + +### 5.3 明确提交结果 + +从当前 `SubmitStatsTask` 抽取内部可区分结果的提交函数，兼容现有 `error` 接口；结果名称以下为拟定值： + +| 结果 | 含义 | pending 处理 | +|---|---|---| +| `Submitted` | 任务元数据已持久化，已交给 scheduler | 确认当前 generation；并发新事件仍保留 | +| `Existing` | 同 segment/subjob 的任务已存在，包括待清理的终态任务 | 不创建副本；任务结果/清理通知负责后续检查，补偿兜底 | +| `NotNeeded` | 最新状态当前不需要构建，或对象已确认删除 | 确认当前 generation；未来资格变化重新唤醒 | +| `Deferred` | 提交容量不足、依赖暂未就绪、collection 信息暂不可用等 | 保留 key，延迟重试，不依赖另一次业务事件 | +| 返回错误 | ID 分配、文件资源查询、任务持久化等失败 | 按现有错误契约处理，必要时退避重试并记录原因 | + +元数据 cache miss 与确认删除必须区分。文件资源仍按当前 schema 和 ref-mode 语义获取，不能因为改成单 segment 处理就省略资源依赖。 + +当前 `AddStatsTask` 的锁以 taskID 为 key，而重复判定使用 segment/subjob。不能仅凭该锁宣称并发唯一性已经保证。第一版发现提交保持串行，审计所有调用者；若引入多提交者，必须先补齐同 segment/subjob 的串行化，并覆盖“检查—分配 ID—持久化—发布”的整个区间。 + +### 5.4 限流与失败重试 + +- 准入不足时，以延迟队列重试已知 key；退避有上限和抖动，禁止立即 requeue + 立即唤醒形成忙循环。 +- 第一版不为了唤醒加入新的 GlobalScheduler 接口；可用本地最早重试定时器。后续若增加容量释放通知，仍需保留定时重试以免丢唤醒。 +- scheduler 已接管的任务不重新分配 taskID，不复制 worker 失败重试状态。 +- schema 在旧任务执行期间变化时，不覆盖旧任务；保留新的检查需求，结果发布和旧任务清理后按最新资格判定是否补建。 +- 禁用 JSON 后停止新增 JSON 任务；不顺带取消此前已接管的任务。再次启用必须主动补偿，不能等待下次 flush。 + +### 5.5 控制 schema 解析成本 + +保留当前未提交的无 map 字段检查优化，先按本次处理批次复用字段 ID 列表。暂不为所有 collection 新建一份常驻 schema 索引。 + +若 profile 仍显示事件批次内反复解析显著，再增加有界字段判定缓存。key 至少覆盖 schema 版本/规则 epoch；external、文件资源模式和 JSON 开关等依赖也需正确失效。缓存只存判定所需信息，不持有完整 schema；命中只是优化，不参与正确性保证。 + +## 6. 补偿扫描与生命周期 + +### 6.1 为什么不能完全删除扫描 + +进程可能在“元数据提交成功”和“记录本地事件”之间退出；通知有容量上限；历史数据和编译期 JSON 格式升级也没有新的业务事件。因此本方案保证的是 **事件优先 + 元数据最终收敛**，不是 exactly-once 事件投递。 + +三类补偿共用预算：启动补偿、溢出/规则变更触发的补偿、低频定时补偿。 + +### 6.2 扫描必须真正有界 + +当前 `GetCollections()` 会物化 collection slice；`CachedSegmentsInfo.GetSegmentsBySelector()` 会先生成 candidates，再生成结果；`ConcurrentMap` 基于 `sync.Map`，没有现成的稳定分页 cursor。 + +因此不能把全量 `GetSegments()/Values()/Keys()` 的结果每次取 100 条就称为有界扫描，也不能反复 `Range` 前 N 项冒充分页。 + +推荐先实现 **单次遍历、分批输出的流式扫描**：一个扫描者依次产出小批 ID，消费者预算不足时暂停推进；不持有业务元数据锁跨批等待，不在内存积累全量候选，不为本改造新增一套每 segment 的常驻索引。collection 扩展同样遵守该约束。 + +实现前必须验证所用遍历器的并发、取消和暂停行为；`sync.Map.Range` 不是一致性快照，不能宣称具有稳定分页或严格扫描耗时上界。新插入/删除由事件与下一轮补偿收敛；如现有遍历方式无法满足锁等待和内存预算，再单独评审更换迭代方式，不暗中引入全量快照缓存。 + +批大小、每秒扫描量和每次处理时间预算必须同时约束“检查过的条目”与“提交的任务”，不能只限制成功提交数量。扫描不因新的全局 dirty 标志不断从头重启；先完成当前轮，再处理新的补偿代数，防止后半部分长期饿死。 + +### 6.3 队列满 + +segment key 无法入队时，非阻塞地将 collection 标记为需补偿；collection dirty 集合也满时，提升为合并的全局补偿 generation。生产者不能等待慢扫描、ID 分配或任务提交。 + +补偿 generation 与普通 key 一样按代确认：本轮扫描期间再次溢出，完成时不能清掉新补偿需求。溢出只安排有界扫描，不在事件回调中同步全扫。 + +在元数据规模有限、扫描持续推进且任务依赖恢复的前提下，漏通知的合格任务应能最终被发现；下游持续饱和时不承诺固定完成时延。须通过扫描完成时间和 oldest-dirty-age 检测无法收敛的状态。 + +### 6.4 启停顺序 + +1. 元数据基础设施就绪后，安装实例级 dirty 收集器，再开放相关正常写入路径，避免扫描与事件订阅之间的空窗。 +2. 恢复已持久化的 Init/Retry/InProgress 任务，仍由原 scheduler 接管。 +3. 启动消费者并安排一轮有界补偿；启动补偿与正常事件合并去重，不同时启动多个全库扫描。 +4. Stop 时禁用/解除回调，取消扫描、等待工作 goroutine 退出；持久化任务继续作为下次恢复依据。 + +配置监听使用独立 handler 标识并 `Unwatch`；不能通过覆盖 ParamItem 的单 callback 影响其他模块。测试须覆盖同进程多次创建/停止 Inspector，确保无旧回调和 goroutine 泄漏。 + +## 7. 分阶段修改与文件范围 + +| 阶段 | 主要工作与拟修改文件 | 进入下一阶段的条件 | +|---|---|---| +| P0：基线与源头审计 | 对 `stats_inspector.go`、`stats_task_meta.go`、`meta.go` 及第 4 节入口建立资格/通知矩阵；补行为测试和 benchmark | 所有资格构造/重写点分类；已有补丁与本改造基线分开；确定队列、扫描预算和发现时延目标 | +| P1：局部 reconcile 和有界基础设施 | 抽取单 segment 判定/提交结果；拟新增 `stats_reconcile_queue.go`；在 `meta.go` / `segment_info_cache.go` 附近提供有界遍历适配 | 去重、处理中再变更、准入不足、取消、扫描内存和公平性测试通过；旧发现模式仍为默认 | +| P2：事件接线与影子验证 | 元数据发布后、collection schema/cache、任务清理和配置变更接入通知；`server.go` 管理生命周期 | 第 4 节所有路径逐项端到端验证；重复/丢通知/重启/rollback 测试通过 | +| P3：切换正常发现路径 | `stats_inspector.go` 从高频全量发现切到事件/pending；增加独立补偿周期和观测；参数定义在 `pkg/util/paramtable/component_param.go`，配置说明在 `configs/milvus.yaml` | 正确性门禁通过，A/B 显示分配和非 idle GC 成本改善，无任务遗漏/饥饿/明显时延回退 | +| P4：IndexInspector 独立后续 | `index_inspector.go`、`go_channel_singleton.go`、`ddl_callbacks_create_index.go` 及必要元数据入口 | Stats 结果先验收；重新 profile 确认 Index 仍值得投入，再复用已验证的有界机制 | + +上述新增文件名和配置名均为计划，不代表当前已经存在。各阶段单独评审；不把现有其他性能补丁顺手提交，不改公共 proto、生成文件或 CI 流程。 + +Index 后续的具体目标：补齐 collection CreateIndex 通知满时的补偿（当前 `default` 会丢通知）、把 overflow 和定时全量扫描改成有界过程、限制 `pendingIndexSegs` 及一次 pop 的规模。保留当前按 segment/collection 触发和任务语义，不另起一套执行 scheduler。 + +## 8. 测试与验收 + +### 8.1 正确性测试矩阵 + +| 场景 | 必须验证的结果 | +|---|---| +| 正常 flush → sort → stats | 每个阶段按最新资格决定，任务结果与旧发现逻辑一致 | +| 同 key 重复通知、乱序通知 | 不创建重复有效任务；旧状态不覆盖新状态 | +| 消费期间再次修改 schema/manifest | 新 generation 不被旧消费完成清掉；最终处理新需求 | +| segment 和 collection 队列同时满 | 生产者及时返回；补偿推进，且历史尾部条目不饿死 | +| Stats 提交队列满，此后无业务事件 | 容量恢复后仍提交；不能因为此前返回 `nil` 永久漏任务 | +| 文件资源/ID 分配/任务持久化失败 | 未持久化的任务不会被确认已提交；依赖恢复后按原错误契约重试 | +| worker 失败/backoff | 由现有 scheduler 重试同一任务，不创建任务风暴 | +| 旧任务 Finished/Failed 尚未清理 | 不重复提交；清理成功后可以发现仍缺少的 stats | +| JSON 关闭 → 开启、旧 kill switch、格式升级 | 关闭语义不变；开启/升级主动补偿全部适用历史数据 | +| external V2/V3、manifest 缺失或更新 | 当前支持范围不变；状态变为合格后能重新检查 | +| Import commit/abort、snapshot restore、schema bump | 覆盖真实元数据来源，且不改变现有可见性/资格规则 | +| Drop/truncate 与旧事件、旧 revision 并发 | 不复活已删除资源，不绕过 tombstone；临时 cache miss 不误判删除 | +| 提交成功后、发事件前 crash；任务持久化后、enqueue 前 crash | 重启通过元数据和任务恢复补齐，无永久遗漏 | +| 多 collection、Text/JSON 大小 backlog 混合 | 有界推进，不被单一 collection 或 subjob 长期独占 | +| 多次 Start/Stop、补偿中取消、配置回调与停止并发 | 无泄漏、无死锁、无旧实例继续提交 | + +Go 行为测试按仓库要求执行，以下是实施阶段的命令，本轮未执行也不代表已有新测试： + +```bash +go test -tags dynamic,test -gcflags="all=-N -l" -count=1 ./internal/datacoord/... +go test -race -tags dynamic,test -gcflags="all=-N -l" -count=1 ./internal/datacoord/... -run 'Stats|Reconcile' +``` + +若触及共享 ParamTable、field helper、cache 或 scheduler，补跑对应包及相关依赖测试；`pkg` 有独立 go.mod，需要在该模块执行。涉及错误分类/封装时，实施前另读 error handling guide/casebook，不在本计划中顺带改变错误码。 + +### 8.2 性能验证 + +至少对比三组：部署旧逻辑、仅字段去分配补丁、字段补丁加事件发现。不能将前两组的差额算作事件改造收益。 + +固定数据规模、schema 分布、Go/构建方式、GOGC、CPU 配额和负载，覆盖无变化、多次 load+search、持续 flush/compaction、schema 批量变化、冷启动和容量饱和场景。性能测试使用一致的生产构建，不能把关闭优化的 mockey 测试构建与生产 profile 直接比较。 + +检查： + +- 分配 bytes/s、objects/s，以及 Stats/Index 路径分配；重复窗口必须覆盖完整补偿周期。 +- GC assist、dedicated/fractional mark、idle mark 分开看，并用 CPU 秒/墙钟秒比较绝对成本，不只看火焰图占比。 +- 稳态和补偿中的 RSS/Go heap、pending 上限、schema 缓存占用及扫描临时内存。 +- 从“元数据已可读且具备资格”到“任务持久化”的 discovery delay，以及任务最终完成时间；两者分开。 +- 队列溢出、最老 dirty 年龄、完整补偿一轮的时长、补偿发现的遗漏任务、两类 subjob 和 collection 公平性。 +- 在可比 load+search 压测中检查查询 p95/p99；不根据后台分配下降直接宣称搜索 latency 已改善。 + +硬性验收：静态数据稳定后，普通唤醒/重试不触发全库枚举；队列和缓存不随运行时间无界增长；故障恢复后无永久遗漏；补偿确实分批推进。分配降幅和发现时延的数值目标在 P0 依据补丁后基线与产品 SLO 确定，不能先承诺“GC 降低一半”。 + +### 8.3 最小观测集 + +复用 `milvus_datacoord_task_count`、`milvus_datacoord_task_num_in_scheduler` 观察执行任务。拟新增发现侧的 pending 数、oldest dirty age、处理结果计数、deferred/overflow 计数、补偿检查数和轮次耗时。 + +新增标签只使用有界的 subjob、原因、结果、扫描类型，不加 collectionID、segmentID、schema version 等无界标签；scrape 不触发全量扫描。日志使用 `mlog`、真实 context 和限频输出，不记录每个重复事件的完整元数据。 + +## 9. 灰度与回滚 + +建议拟新增启动级模式 `dataCoord.statsInspector.discoveryMode = poll | shadow | event`，默认 `poll`。补偿周期独立配置，不改变共享 `TaskCheckInterval`。具体容量/批量默认值在 P0 测量后确定，参数非法值在启动阶段拒绝。 + +- `poll`：保留现有正常发现；新机制不得增加后台全库扫描。 +- `shadow`：旧路径提交任务，新路径只做只读资格检查/覆盖抽样，不分配任务 ID、不持久化和 enqueue。该模式有额外开销，只用于受控验证。 +- `event`：新路径提交任务，关闭旧的高频全量发现，保留有界补偿。 + +影子差异应基于同一元数据版本或可重放测试快照比较。实时并发下，任务已被另一条路径提交、schema 更新等导致的瞬时差异不能直接判为遗漏；若无法取得一致输入，只能作为诊断证据，不能作为正确性证明。 + +先在测试环境做事件丢失/重启故障注入，再在授权的实例灰度。出现任务长期未发现、补偿无法收敛、队列/RSS 无界增长或时延回退即停止扩大范围，切回 `poll` 并重启组件。复用原任务格式，回滚从原元数据恢复；不需要删除任务、回退 schema 或重写 WAL。部署与重启需要另行授权。 + +## 10. 开始编码前的决策门槛 + +1. 完成第 4 节所有资格变化源头和提交出口审计，确认没有只发 index 通知而漏 stats 的路径。 +2. 明确队列总容量、扫描吞吐预算、最大补偿延迟和发现时延 SLO；这些是负载相关参数，不凭默认值推断。 +3. 验证有界流式遍历的真实内存/锁行为；若要新增常驻全量索引，必须另行评审内存代价。 +4. 确认无新字段/功能被这次改造启用，schema/任务生命周期的资格判定与旧路径一致。 + +建议先实施 P0/P1，通过后再接入全部事件。未满足源头审计与故障路径验证前，不删除旧扫描，也不宣称改造已经解决 GC 或搜索延迟问题。 + +## 附录：证据与参考 + +- 可复现的分配基准：[Stats 发现 benchmark](../../../internal/datacoord/stats_discovery_benchmark_test.go)、[字段 helper benchmark](../../../pkg/util/typeutil/field_match_test.go)。 +- 关键实现：[segment cache](../../../internal/datacoord/segment_info_cache.go)、[versioned cache](../../../internal/datacoord/write_through_cache.go)、[GlobalScheduler](../../../internal/datacoord/task/global_scheduler.go)、[字段 helper](../../../pkg/util/typeutil/field_schema.go)、[参数定义](../../../pkg/util/paramtable/component_param.go)。 +- 约束参考：[Streaming System](../../../docs/agent_guides/streaming-system/streaming-system.md)、[Broadcaster](../../../docs/agent_guides/streaming-system/coordination/broadcaster.md)、[Collection 消息语义](../../../docs/agent_guides/streaming-system/message/message-semantic-collection.md)、[Observability](../../../docs/agent_guides/observability/README.md)。本文不改变这些文档定义的协议边界。 + +本地运维采样证据不随 PR 上传；本文的仓库链接均为相对路径。 + +## 实施记录（2026-09-08） + +实施基线：`codex/load-1m-segments-pr-stack-rebased-qv-work`，`e99fe0e5e1f31fc8c84978fea98180b84a0b1f81`。 +提交基线：`7e32a5ca9b7e472f1563668e497595a1f57b0d7d`。工作分支：`codex/stats-inspector-event-driven`。原工作目录未修改；其他已回退补丁未恢复。 + +### 实际结构 + +- `stats_reconcile_queue.go`：实例级有界 dirty 状态；Text/JSON 各一个最早重试堆，在两者都 ready 时轮换；同 key 去重，完成时按 generation 确认。 +- `stats_reconcile.go`：单 segment 最新状态检查、明确提交结果、批内字段/文件资源复用、配置订阅、流式补偿。 +- `stats_discovery_meta.go`：只对资格/依赖字段变化发通知，忽略行数等无关更新；不把完整元数据装入队列。 +- 元数据通知均在成功提交并发布 cache 后发生；事件是提示，消费者重新读取 cache，沿用原有 revision/tombstone 保护。 +- 单次持续遍历通过 `iter.Pull2` 暂停，不调用全量 `Values/GetSegments` 生成快照。全库扫描直接访问原 cache 的 entries，连 tombstone 都按条计入预算，避免在底层过滤中隐含无界遍历。没有新增全量 segment 索引。 +- 元数据依赖 RPC 可以超过本地 10 ms 处理时间片，但有 context 取消/超时；不承诺单个时间片的严格墙钟上界。 +- 只更改 Text/JSON 任务发现；已有 worker/scheduler 重试、任务回收、BM25 禁用、Sort compaction、Load/Search 协议保持原样。恢复时已有 Sort stats 任务仍由旧逻辑跳过，不新增 Sort stats 执行。 + +### 配置与内部预算 + +只新增两个公开、启动时生效的配置： + +```yaml +dataCoord: + statsInspector: + discoveryMode: poll + reconcileInterval: 600 +``` + +`poll` 仍是默认：不创建事件队列/消费者/额外补扫；`shadow` 的新路径不分配任务 ID、不持久化、不 enqueue，旧路径继续提交；`event` 关闭原发现 ticker。 +`reconcileInterval` 单位秒，范围 1–86400；模式非法值也在初始化时拒绝。 + +内部初始预算：4096 个 key（含处理中和延迟项），单 collection 最多占 1/8；128 个 collection 扫描范围加一个合并的全局范围；最多 4 个暂停的遍历器共享每 100 ms 128 条的扫描预算。每轮消费最多 64 个 key，10 ms 软时间片；发现重试为 1–30 秒有界退避和抖动。容量和批量是内部选项，不再暴露六个独立调参项。 + +这些是本地验证使用的保守预算，不是负载 SLO。扫描理想上限为 1280 条/s，百万条至少约 13 分钟，仍会受 tombstone、任务容量和依赖延迟影响。完整扫描不因周期到期而重启；扫描期间的实际变更保留下一代补偿需求。 + +### 通知源审计 + +| 原始入口/状态变化 | 实际发布出口 | 验证方式 | +|---|---|---| +| Flush 状态、manifest 回调、Import commit/abort 和结果、external refresh、stats 结果 | `SetState/UpdateSegment/UpdateSegmentsInfo` | 公共出口成功/失败/无关更新测试；审计业务调用链 | +| 导入新 segment、snapshot restore | `AddSegment` | 持久化失败不发通知、成功发通知；源码追踪 restore/import | +| Mix/Sort/Clustering/schema bump（含原 ID 原地更新） | `CompleteCompactionMutation` 公共包装层 | 四种 compaction 的真实内存持久化路径测试 | +| 删除、分区 drop、channel drop、truncate、批量 GC 删除 | 对应 `meta` 发布出口 | 删除后过期事件、不复活 tombstone、批量删除通知测试 | +| Schema/外部字段映射/文件资源 ID 变化 | `AddCollection`，仅 schema 有变化时安排 collection 补扫 | 新字段通知、资源查询期间 schema 替换测试 | +| 旧任务终态清理 | `DropStatsTask` 成功移除任务和二级索引之后 | 清理失败保留占位；成功后重新发现 | +| JSON 开关、旧 kill switch、文件资源模式 | ParamTable `Watch/Unwatch` | 开关主动补扫及旧实例回调解除测试 | +| 启动/重启/漏通知 | 初次有界补扫、原 `reloadFromMeta` | 漏通知、持久化任务恢复、不重新分配已有 taskID 测试 | + +生产 `AddStatsTask` 调用者审计后只有 Inspector。提交 mutex 覆盖检查、ID 分配、持久化和 enqueue，不能把 taskID 锁误当作 segment/subjob 唯一性锁。 +`CachedSegmentsInfo` 本地更新只涉及行数、position、allocation、compacting 等，不改变 Stats 资格；不接入这些高频更新。 +collection cache miss 可重试，只有现有 handler 确认 CollectionNotFound 才结束检查。external 判定沿用 `typeutil.IsExternalCollection` 的外部字段映射语义，不能仅在测试中设置 ExternalSource 就当成 external collection。 + +### 已完成的本地验证与边界 + +- 新增队列、发现、资源失败、容量恢复、漏通知、schema/config、终态清理、取消、重复启停、并发提交、四类 compaction 测试；原 StatsInspector/StatsTaskMeta 一同验证。还覆盖任务已持久化但未 enqueue 就重启的恢复窗口，确认复用已有 taskID。 +- 上述 focused 测试在提交基线 `7e32a5ca9` 上以 `-race -tags dynamic,test -gcflags='all=-N -l' -count=3` 连续通过，最后一轮耗时 16.039 秒。DataCoord 和涉及的 pkg 包增量 `golangci-lint --new-from-rev=7e32a5ca9` 均为 0 issues。 +- 资格用同一份元数据对照旧 trigger：Flushing/Flushed、L0、sorted/namespace sorted、external V2/V3、当前 JSON 格式、Text/JSON，以及不新增 BM25/Sort。 +- 配置默认值/非法值/启动级属性、字段 helper 等价性和零分配路径、指标注册测试。 +- 全量 DataCoord 与共享包测试已尝试，并非全绿。未修改的基线复现了提交时间戳断言、DISKANN 大小断言、external refresh/schema bump 测试缺少持久化对象导致的 panic、Meta reload 缺少 mock 预期、QueryView 配置断言及 Proxy 指标旧标签数导致的 panic。未顺手修改这些不相关测试或业务逻辑。 +- 单独的本地 etcd/MinIO 用于回归；不部署、不变更线上配置。提交 PR 不代表完成上线验收。 +- 提交前在独立验证工作树执行 `make lint-fix`，gofumpt/gci 已运行，根模块 typecheck 停在 `internal/metastore/kv/querycoord/kv_catalog_test.go:372: undefined: mocks`。该文件与提交基线的 blob 完全相同；本次不修改，pkg/client 后续 lint 阶段未执行。固定版本 gci 会将标准库 `iter` 错分到第三方组，因此保留已通过当前增量 lint 的标准库分组；未纳入其他自动格式化改动。 +- `run_clang_format.sh` 使用 clang-format 15.0.7 完成；本 PR 无 C/C++ 文件变更。`git diff --check` 通过。 + +下面的微基准使用同一 Go 1.26.5、darwin/arm64、`-tags dynamic,test -gcflags='all=-N -l'`，不是生产优化构建，也不是线上 A/B： + +| 场景 | 1000 segment | 100000 segment | +|---|---:|---:| +| 旧周期发现（已使用无 map 字段检查） | 51,744 B/次 | 10,624,400 B/次 | +| 事件模式无待处理变化的一次消费 | 0 B/次 | 0 B/次 | +| 创建流式游标、读取第一项并停止 | 236 B/次，7 次分配 | 232 B/次，7 次分配 | + +重复 key 通知：0 B/次；缺失 enable_match 字段判断：旧 helper 617 B/次、12 次分配，新直接判断 0 B/次。 +这些只验证局部成本不再随存量扫描放大；事件补扫、实际新任务仍有成本，不能解释成整个进程零分配，不能换算为 GC CPU 或 search p99 降幅。 + +### 尚待环境验证 + +- 未执行完整的 `make test-go`（包含重新生成 proto、重建 C++ 等前置步骤）；已尝试的 DataCoord/共享包全量 Go 测试本身也未全绿，不能将 focused/race 通过称为完整 CI 成功。 +- 生产构建和同负载 A/B，覆盖完整补偿周期、持续 flush/compaction、大 schema 变更、容量饱和、多 collection；确认扫描可收敛及业务发现时延 SLO。 +- 部署、启用 event、线上 CPU/alloc pprof 和 load+search p95/p99 对比需要另行授权。默认 poll 保留用于回滚。 diff --git a/internal/datacoord/meta.go b/internal/datacoord/meta.go index fde95a9a659..f6522e55760 100644 --- a/internal/datacoord/meta.go +++ b/internal/datacoord/meta.go @@ -102,6 +102,7 @@ type meta struct { segments *CachedSegmentsInfo // segment id to segment info dataViewManager DataViewManager queryViewLoadInfoNotifier QueryViewLoadInfoNotifier + statsDiscovery atomic.Pointer[statsReconcileQueue] channelCPs *channelCPs // vChannel -> channel checkpoint/see position chunkManager storage.ChunkManager @@ -744,7 +745,11 @@ func (m *meta) addCollectionToCache(collection *collectionInfo) { // Note that collection info is just for caching and will not be set into etcd from datacoord func (m *meta) AddCollection(collection *collectionInfo) { mlog.Info(context.TODO(), "meta update: add collection", zap.Int64("collectionID", collection.ID)) + old := m.GetCollection(collection.ID) m.addCollectionToCache(collection) + if q := m.statsDiscovery.Load(); q != nil && (old == nil || !proto.Equal(old.Schema, collection.Schema)) { + q.requestScan(collection.ID, true) + } metrics.DataCoordNumCollections.WithLabelValues().Set(float64(m.collections.Len())) mlog.Info(context.TODO(), "meta update: add collection - complete", zap.Int64("collectionID", collection.ID)) } @@ -753,6 +758,9 @@ func (m *meta) AddCollection(collection *collectionInfo) { func (m *meta) DropCollection(collectionID int64) { mlog.Info(context.TODO(), "meta update: drop collection", zap.Int64("collectionID", collectionID)) if _, ok := m.collections.GetAndRemove(collectionID); ok { + if q := m.statsDiscovery.Load(); q != nil { + q.requestScan(collectionID, true) + } metrics.CleanupDataCoordWithCollectionID(collectionID) metrics.DataCoordNumCollections.WithLabelValues().Set(float64(m.collections.Len())) mlog.Info(context.TODO(), "meta update: drop collection - complete", zap.Int64("collectionID", collectionID)) @@ -1040,6 +1048,7 @@ func (m *meta) AddSegment(ctx context.Context, segment *SegmentInfo) error { return err } m.segments.SetSegment(segment.GetID(), segment, results[0].Version) + m.notifyStatsChange(nil, segment) metrics.DataCoordNumSegments.WithLabelValues(segmentMetricLabelValues(segment)...).Inc() logger.Info(ctx, "meta update: adding segment - complete", zap.Int64("segmentID", segment.GetID())) @@ -1059,6 +1068,7 @@ func (m *meta) DropSegment(ctx context.Context, segment *SegmentInfo) error { if errors.Is(err, ErrKeyNotFound) { logger.Info(ctx, "meta update: dropping segment - already deleted", zap.Int64("segmentID", segmentID)) m.segments.DropSegment(segmentID, math.MaxInt64) + m.notifyStatsChange(segment, nil) return nil } logger.Warn(ctx, "meta update: dropping segment failed", @@ -1069,6 +1079,7 @@ func (m *meta) DropSegment(ctx context.Context, segment *SegmentInfo) error { metrics.DataCoordNumSegments.WithLabelValues(segmentMetricLabelValues(segment)...).Dec() m.segments.DropSegment(segmentID, results[0].Version) + m.notifyStatsChange(segment, nil) logger.Info(ctx, "meta update: dropping segment - complete", zap.Int64("segmentID", segmentID)) return nil @@ -1138,6 +1149,7 @@ func (m *meta) DropSegments(ctx context.Context, candidates []*SegmentInfo) (int if singleErr != nil { if errors.Is(singleErr, ErrKeyNotFound) { m.segments.DropSegment(segment.GetID(), math.MaxInt64) + m.notifyStatsChange(segment, nil) removed++ continue } @@ -1152,6 +1164,7 @@ func (m *meta) DropSegments(ctx context.Context, candidates []*SegmentInfo) (int } metrics.DataCoordNumSegments.WithLabelValues(segmentMetricLabelValues(segment)...).Dec() m.segments.DropSegment(segment.GetID(), singleResults[0].Version) + m.notifyStatsChange(segment, nil) removed++ } return removed, deleteErr @@ -1167,6 +1180,7 @@ func (m *meta) DropSegments(ctx context.Context, candidates []*SegmentInfo) (int for i, segment := range segments { metrics.DataCoordNumSegments.WithLabelValues(segmentMetricLabelValues(segment)...).Dec() m.segments.DropSegment(segment.GetID(), results[i].Version) + m.notifyStatsChange(segment, nil) } return len(segments), nil } @@ -1277,6 +1291,7 @@ func (m *meta) SetState(ctx context.Context, segmentID UniqueID, targetState com } updatedSeg := NewSegmentInfo(results[0].Value) old, existed := m.segments.SetSegment(segmentID, updatedSeg, results[0].Version) + m.notifyStatsChange(old, updatedSeg) if existed && old.GetState() != updatedSeg.GetState() { metricMutation := segMetricMutation{stateChange: make(segmentMetricStateChange)} metricMutation.appendSegmentLabelChange(old, updatedSeg) @@ -1318,7 +1333,9 @@ func (m *meta) UpdateSegment(segmentID int64, operators ...SegmentOperator) erro return err } // Update in-memory meta. - m.segments.SetSegment(segmentID, NewSegmentInfo(results[0].Value), results[0].Version) + updated := NewSegmentInfo(results[0].Value) + old, _ := m.segments.SetSegment(segmentID, updated, results[0].Version) + m.notifyStatsChange(old, updated) logger.Info(context.TODO(), "meta update: update segment - complete", zap.Int64("segmentID", segmentID)) @@ -1692,7 +1709,8 @@ func (m *meta) UpdateSegmentsInfo(ctx context.Context, mutations map[int64][]Mut type entry struct { segID int64 isInsert bool - newSeg *SegmentInfo // only for inserts + newSeg *SegmentInfo // published insert/update value + oldSeg *SegmentInfo } var entries []entry @@ -1764,6 +1782,8 @@ func (m *meta) UpdateSegmentsInfo(ctx context.Context, mutations map[int64][]Mut } else { newSeg := NewSegmentInfo(results[i].Value) oldSeg, existed := m.segments.SetSegment(e.segID, newSeg, results[i].Version) + entries[i].oldSeg = oldSeg + entries[i].newSeg = newSeg if existed && !sameSegmentMetricLabels(oldSeg, newSeg) { metricMutation.appendSegmentLabelChange(oldSeg, newSeg) } @@ -1771,6 +1791,10 @@ func (m *meta) UpdateSegmentsInfo(ctx context.Context, mutations map[int64][]Mut } metricMutation.commit() cacheDur := time.Since(cacheStart) + // Publish hints only once the whole committed batch is visible in cache. + for _, e := range entries { + m.notifyStatsChange(e.oldSeg, e.newSeg) + } totalDur := time.Since(start) if totalDur > 40*time.Millisecond { @@ -1929,6 +1953,10 @@ func (m *meta) UpdateDropChannelSegmentInfo(ctx context.Context, channel string, } metricMutation.commit() + // All cache entries in this transaction have been published. + for _, result := range results { + m.notifyStatsSegments(result.Value.GetCollectionID(), result.Value.GetID()) + } logger.Info(ctx, "meta update: update drop channel segment info - complete", zap.String("channel", channel)) return nil @@ -2019,7 +2047,6 @@ func (m *meta) GetFlushingSegments() []*SegmentInfo { // SelectSegments select segments with selector func (m *meta) SelectSegments(ctx context.Context, filters ...SegmentFilter) []*SegmentInfo { - return m.segments.GetSegmentsBySelector(filters...) } @@ -2049,7 +2076,6 @@ func (m *meta) GetCollectionIDsByPartition(ctx context.Context, partitionIDs []i } func (m *meta) GetRealSegmentsForChannel(channel string) []*SegmentInfo { - return m.segments.GetRealSegmentsForChannel(channel) } @@ -2073,14 +2099,12 @@ func (m *meta) AddAllocation(segmentID UniqueID, allocation *Allocation) error { } func (m *meta) SetRowCount(segmentID UniqueID, rowCount int64) { - m.segments.SetRowCount(segmentID, rowCount) } // SetAllocations set Segment allocations, will overwrite ALL original allocations // Note that allocations is not persisted in KV store func (m *meta) SetAllocations(segmentID UniqueID, allocations []*Allocation) { - m.segments.SetAllocations(segmentID, allocations) } @@ -2093,26 +2117,22 @@ func (m *meta) SetLastExpire(segmentID UniqueID, lastExpire uint64) { // SetLastFlushTime set LastFlushTime for segment with provided `segmentID` // Note that lastFlushTime is not persisted in KV store func (m *meta) SetLastFlushTime(segmentID UniqueID, t time.Time) { - m.segments.SetFlushTime(segmentID, t) } // SetLastWrittenTime set LastWrittenTime for segment with provided `segmentID` // Note that lastWrittenTime is not persisted in KV store func (m *meta) SetLastWrittenTime(segmentID UniqueID) { - m.segments.SetLastWrittenTime(segmentID) } // SetSegmentCompacting sets compaction state for segment func (m *meta) SetSegmentCompacting(segmentID UniqueID, compacting bool) { - m.segments.SetIsCompacting(segmentID, compacting) } // IsSegmentCompacting check if segment is compacting func (m *meta) IsSegmentCompacting(segmentID UniqueID) bool { - seg := m.segments.GetSegment(segmentID) if seg == nil { return false @@ -2124,7 +2144,6 @@ func (m *meta) IsSegmentCompacting(segmentID UniqueID) bool { // if true, set them compacting and return true // if false, skip setting and func (m *meta) CheckAndSetSegmentsCompacting(ctx context.Context, segmentIDs []UniqueID) (exist, canDo bool) { - var hasCompacting bool exist = true for _, segmentID := range segmentIDs { @@ -2148,7 +2167,6 @@ func (m *meta) CheckAndSetSegmentsCompacting(ctx context.Context, segmentIDs []U } func (m *meta) SetSegmentsCompacting(ctx context.Context, segmentIDs []UniqueID, compacting bool) { - for _, segmentID := range segmentIDs { m.segments.SetIsCompacting(segmentID, compacting) } @@ -2470,7 +2488,6 @@ func (m *meta) completeMixCompactionMutation( } func (m *meta) ValidateSegmentStateBeforeCompleteCompactionMutation(t *datapb.CompactionTask) error { - if t.GetType() != datapb.CompactionType_Level0DeleteCompaction { if m.isCollectionCompactionBlocked(t.GetCollectionID()) { mlog.Info(context.TODO(), "compaction rejected: collection has pending snapshot or unloaded RefIndex", @@ -2541,6 +2558,10 @@ func (m *meta) CompleteCompactionMutation(ctx context.Context, t *datapb.Compact m.publishDataViewAfterCompaction(ctx, t, lo.Map(newSegments, func(segment *SegmentInfo, _ int) int64 { return segment.GetID() })) + m.notifyStatsSegments(t.GetCollectionID(), t.GetInputSegments()...) + for _, segment := range newSegments { + m.notifyStatsSegments(segment.GetCollectionID(), segment.GetID()) + } return newSegments, metricMutation, nil } @@ -2583,7 +2604,6 @@ func isSegmentHealthy(segment *SegmentInfo) bool { } func (m *meta) HasSegments(segIDs []UniqueID) (bool, error) { - for _, segID := range segIDs { if m.segments.GetSegment(segID) == nil { return false, fmt.Errorf("segment is not exist with ID = %d", segID) @@ -2594,7 +2614,6 @@ func (m *meta) HasSegments(segIDs []UniqueID) (bool, error) { // GetCompactionTo returns the segment info of the segment to be compacted to. func (m *meta) GetCompactionTo(segmentID int64) ([]*SegmentInfo, bool) { - return m.segments.GetCompactionTo(segmentID) } @@ -3395,7 +3414,6 @@ func (m *meta) completeBumpSchemaVersionReplacementMutation( } func (m *meta) getSegmentsMetrics(collectionID int64) []*metricsinfo.Segment { - allSegments := m.segments.GetSegments() segments := make([]*metricsinfo.Segment, 0, len(allSegments)) for _, s := range allSegments { @@ -3421,7 +3439,6 @@ func (m *meta) getSegmentsMetrics(collectionID int64) []*metricsinfo.Segment { } func (m *meta) DropSegmentsOfPartition(ctx context.Context, partitionIDs []int64) error { - // Collect segments to drop (read-only from cache for key construction). type segRef struct { id int64 @@ -3463,6 +3480,9 @@ func (m *meta) DropSegmentsOfPartition(ctx context.Context, partitionIDs []int64 } } metricMutation.commit() + for _, result := range results { + m.notifyStatsSegments(result.Value.GetCollectionID(), result.Value.GetID()) + } return nil } @@ -3484,7 +3504,6 @@ func (m *meta) GetFileResources(ctx context.Context, resourceIDs ...int64) ([]*i // TruncateChannelByTime drops segments of a channel that were updated before the flush timestamp func (m *meta) TruncateChannelByTime(ctx context.Context, vChannel string, flushTs uint64) error { - segments := m.segments.GetSegmentsBySelector(SegmentFilterFunc(isSegmentHealthy), WithChannel(vChannel)) // Collect segments to drop (read-only from cache for key construction and filtering). @@ -3533,6 +3552,9 @@ func (m *meta) TruncateChannelByTime(ctx context.Context, vChannel string, flush } } metricMutation.commit() + for _, result := range results { + m.notifyStatsSegments(result.Value.GetCollectionID(), result.Value.GetID()) + } return nil } diff --git a/internal/datacoord/stats_discovery_benchmark_test.go b/internal/datacoord/stats_discovery_benchmark_test.go new file mode 100644 index 00000000000..dd53ef87852 --- /dev/null +++ b/internal/datacoord/stats_discovery_benchmark_test.go @@ -0,0 +1,60 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "fmt" + "iter" + "testing" + + "github.com/milvus-io/milvus/pkg/v3/common" + "github.com/milvus-io/milvus/pkg/v3/proto/datapb" +) + +func BenchmarkStatsDiscoverySteady(b *testing.B) { + for _, size := range []int{1000, 100000} { + b.Run(fmt.Sprint(size), func(b *testing.B) { + f := newDiscoveryFixture(b, "event") + for id := int64(1); id <= int64(size); id++ { + segment := discoverySegment(id, true) + segment.TextStatsLogs = map[int64]*datapb.TextIndexStats{101: {}} + segment.JsonKeyStats = map[int64]*datapb.JsonKeyStats{102: {JsonKeyStatsDataFormat: common.JSONStatsDataFormatVersion}} + f.mt.segments.SetSegment(id, segment, 1) + } + b.Run("poll_with_direct_field_check", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + f.si.triggerStatsTasks(0) + } + }) + b.Run("event_no_change", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + f.si.processStatsDiscoveryBatch() + } + }) + b.Run("stream_first_entry", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + next, stop := iter.Pull2(f.mt.rangeStatsSegments(0)) + next() + stop() + } + }) + }) + } +} diff --git a/internal/datacoord/stats_discovery_compaction_test.go b/internal/datacoord/stats_discovery_compaction_test.go new file mode 100644 index 00000000000..a65772d6370 --- /dev/null +++ b/internal/datacoord/stats_discovery_compaction_test.go @@ -0,0 +1,72 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/storagev2/packed" + "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" +) + +func TestStatsDiscoveryCompactionPublication(t *testing.T) { + for _, kind := range []datapb.CompactionType{ + datapb.CompactionType_MixCompaction, + datapb.CompactionType_SortCompaction, + datapb.CompactionType_ClusteringCompaction, + datapb.CompactionType_BumpSchemaVersionCompaction, + } { + t.Run(kind.String(), func(t *testing.T) { + input := newCompactionCreateTsTestSegment(1, datapb.SegmentLevel_L1) + input.IsSorted = true + input.SchemaVersion = 1 + manifest := packed.MarshalManifestPath("/test/stats/1", 10) + input.StorageVersion = storage.StorageV3 + input.ManifestPath = manifest + mt := newCompactionCreateTsTestMeta(t, input) + q := newStatsReconcileQueue(64, 4) + mt.statsDiscovery.Store(q) + task := &datapb.CompactionTask{ + CollectionID: 100, InputSegments: []int64{1}, Type: kind, Channel: "ch-1", + Schema: &schemapb.CollectionSchema{Version: 2}, + } + output := int64(2) + if kind == datapb.CompactionType_BumpSchemaVersionCompaction { + output = 1 + } + result := &datapb.CompactionPlanResult{Segments: []*datapb.CompactionSegment{{ + SegmentID: output, NumOfRows: 100, StorageVersion: storage.StorageV3, + Manifest: manifest, BaseManifest: manifest, + InsertLogs: []*datapb.FieldBinlog{getFieldBinlogIDs(0, 20000)}, + Field2StatslogPaths: []*datapb.FieldBinlog{getFieldBinlogIDs(0, 20001)}, + }}} + segments, _, err := mt.CompleteCompactionMutation(context.Background(), task, result) + require.NoError(t, err) + require.Len(t, segments, 1) + require.NotNil(t, mt.GetSegment(context.Background(), output)) + require.Contains(t, q.pending, statsReconcileKey{output, indexpb.StatsSubJob_TextIndexJob}) + require.Contains(t, q.pending, statsReconcileKey{output, indexpb.StatsSubJob_JsonKeyIndexJob}) + require.Contains(t, q.pending, statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob}) + }) + } +} diff --git a/internal/datacoord/stats_discovery_failure_test.go b/internal/datacoord/stats_discovery_failure_test.go new file mode 100644 index 00000000000..9b80d165a48 --- /dev/null +++ b/internal/datacoord/stats_discovery_failure_test.go @@ -0,0 +1,253 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus/internal/datacoord/broker" + "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" + "github.com/milvus-io/milvus/pkg/v3/proto/internalpb" + "github.com/milvus-io/milvus/pkg/v3/util/merr" +) + +type discoveryTestHandler struct { + Handler + get func(context.Context, int64) (*collectionInfo, error) +} + +func (h *discoveryTestHandler) GetCollection(ctx context.Context, id int64) (*collectionInfo, error) { + return h.get(ctx, id) +} + +type discoveryTestBroker struct { + broker.Broker + get func(context.Context, ...int64) ([]*internalpb.FileResourceInfo, error) +} + +func (b *discoveryTestBroker) GetFileResources(ctx context.Context, ids ...int64) ([]*internalpb.FileResourceInfo, error) { + return b.get(ctx, ids...) +} + +func TestStatsDiscoveryCollectionCacheMiss(t *testing.T) { + f := newDiscoveryFixture(t, "event") + col := f.mt.GetCollection(1) + f.mt.collections.Remove(1) + f.mt.segments.SetSegment(1, discoverySegment(1, true), 1) + key := statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob} + result, err := f.si.reconcileStats(key, make(map[int64]statsFieldRules)) + require.NoError(t, err) + require.Equal(t, statsDeferred, result, "cache miss is not proof of deletion") + f.si.handler = &discoveryTestHandler{get: func(context.Context, int64) (*collectionInfo, error) { + return nil, merr.WrapErrServiceInternalMsg("temporary lookup failure") + }} + result, err = f.si.reconcileStats(key, make(map[int64]statsFieldRules)) + require.Error(t, err) + require.Equal(t, statsDeferred, result) + f.si.handler = &discoveryTestHandler{get: func(context.Context, int64) (*collectionInfo, error) { + return nil, merr.WrapErrCollectionNotFound(1) + }} + result, err = f.si.reconcileStats(key, make(map[int64]statsFieldRules)) + require.NoError(t, err) + require.Equal(t, statsNotNeeded, result) + f.si.handler = &discoveryTestHandler{get: func(context.Context, int64) (*collectionInfo, error) { + f.mt.AddCollection(col) + return col, nil + }} + result, err = f.si.reconcileStats(key, make(map[int64]statsFieldRules)) + require.NoError(t, err) + require.Equal(t, statsSubmitted, result) +} + +func TestStatsDiscoveryResourcesAndSchemaRace(t *testing.T) { + f := newDiscoveryFixture(t, "event") + setDiscoveryTestParam(t, &Params.CommonCfg.DNFileResourceMode, "ref") + col := f.mt.GetClonedCollectionInfo(1) + col.Schema.FileResourceIds = []int64{7} + f.mt.AddCollection(col) + f.mt.segments.SetSegment(1, discoverySegment(1, true), 1) + key := statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob} + resource := &internalpb.FileResourceInfo{} + f.mt.broker = &discoveryTestBroker{get: func(context.Context, ...int64) ([]*internalpb.FileResourceInfo, error) { + return nil, merr.WrapErrServiceInternalMsg("temporary resource lookup failure") + }} + result, err := f.si.reconcileStats(key, make(map[int64]statsFieldRules)) + require.Error(t, err) + require.Equal(t, statsDeferred, result) + require.Zero(t, f.alloc.calls.Load()) + f.mt.broker = &discoveryTestBroker{get: func(_ context.Context, ids ...int64) ([]*internalpb.FileResourceInfo, error) { + require.Equal(t, []int64{7}, ids) + changed := f.mt.GetClonedCollectionInfo(1) + changed.Schema.FileResourceIds = []int64{8} + f.mt.AddCollection(changed) + return []*internalpb.FileResourceInfo{resource}, nil + }} + result, err = f.si.reconcileStats(key, make(map[int64]statsFieldRules)) + require.NoError(t, err) + require.Equal(t, statsDeferred, result, "resources fetched for an old schema must not be submitted") + require.Zero(t, f.alloc.calls.Load()) + f.mt.broker = &discoveryTestBroker{get: func(_ context.Context, ids ...int64) ([]*internalpb.FileResourceInfo, error) { + require.Equal(t, []int64{8}, ids) + return []*internalpb.FileResourceInfo{resource}, nil + }} + result, err = f.si.reconcileStats(key, make(map[int64]statsFieldRules)) + require.NoError(t, err) + require.Equal(t, statsSubmitted, result) + st := f.mt.statsTaskMeta.GetStatsTaskBySegmentID(1, key.subjob) + require.Len(t, st.GetFileResources(), 1) + + // Successful resource lookups are shared only within this bounded batch. + calls := 0 + f.mt.broker = &discoveryTestBroker{get: func(context.Context, ...int64) ([]*internalpb.FileResourceInfo, error) { + calls++ + return []*internalpb.FileResourceInfo{resource}, nil + }} + rules := make(map[int64]statsFieldRules) + for _, id := range []int64{2, 3} { + f.mt.segments.SetSegment(id, discoverySegment(id, true), 1) + result, err := f.si.reconcileStats(statsReconcileKey{id, key.subjob}, rules) + require.NoError(t, err) + require.Equal(t, statsSubmitted, result) + } + require.Equal(t, 1, calls) +} + +func TestStatsDiscoveryCancellationDuringDependency(t *testing.T) { + f := newDiscoveryFixture(t, "event") + setDiscoveryTestParam(t, &Params.CommonCfg.DNFileResourceMode, "ref") + col := f.mt.GetClonedCollectionInfo(1) + col.Schema.FileResourceIds = []int64{7} + f.mt.AddCollection(col) + entered := make(chan struct{}) + var once sync.Once + f.mt.broker = &discoveryTestBroker{get: func(ctx context.Context, _ ...int64) ([]*internalpb.FileResourceInfo, error) { + once.Do(func() { close(entered) }) + <-ctx.Done() + return nil, ctx.Err() + }} + require.NoError(t, f.mt.AddSegment(context.Background(), discoverySegment(1, true))) + f.si.Start() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("resource lookup not reached") + } + stopped := make(chan struct{}) + go func() { f.si.Stop(); close(stopped) }() + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("stop did not cancel resource lookup") + } + require.False(t, f.mt.statsTaskMeta.HasStatsTask(1, indexpb.StatsSubJob_TextIndexJob)) +} + +func TestStatsDiscoveryConcurrentSubmission(t *testing.T) { + f := newDiscoveryFixture(t, "event") + f.mt.segments.SetSegment(1, discoverySegment(1, true), 1) + var wg sync.WaitGroup + errs := make(chan error, 16) + for range 16 { + wg.Add(1) + go func() { + defer wg.Done() + errs <- f.si.SubmitStatsTask(1, 1, indexpb.StatsSubJob_TextIndexJob, true, nil) + }() + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + require.EqualValues(t, 1, f.alloc.calls.Load()) + require.EqualValues(t, 1, f.scheduler.enqueued.Load()) +} + +func TestStatsDiscoveryRecoveryAfterPersistBeforeEnqueue(t *testing.T) { + f := newDiscoveryFixture(t, "event") + f.mt.segments.SetSegment(1, discoverySegment(1, true), 1) + // Simulate a process exit after persistence but before scheduler.Enqueue. + require.NoError(t, f.catalog.SaveStatsTask(context.Background(), &indexpb.StatsTask{ + TaskID: 9000, CollectionID: 1, PartitionID: 2, SegmentID: 1, TargetSegmentID: 1, + SubJobType: indexpb.StatsSubJob_TextIndexJob, State: indexpb.JobState_JobStateInit, + })) + reloaded, err := newStatsTaskMeta(context.Background(), f.catalog) + require.NoError(t, err) + f.mt.statsTaskMeta = reloaded + reloaded.statsDiscovery.Store(f.si.discovery) + f.si.Start() + f.waitTasks(t, 2) + require.EqualValues(t, 9000, reloaded.GetStatsTaskBySegmentID(1, indexpb.StatsSubJob_TextIndexJob).GetTaskID()) + require.EqualValues(t, 1, f.alloc.calls.Load(), "only the missing JSON task needs a new ID") +} + +func TestStatsDiscoveryBatchDeleteNotifications(t *testing.T) { + for _, op := range []string{"truncate", "drop_channel", "batch_delete"} { + t.Run(op, func(t *testing.T) { + f := newDiscoveryFixture(t, "event") + segment := discoverySegment(1, false) + require.NoError(t, f.mt.AddSegment(context.Background(), segment)) + drainDiscovery(f.si.discovery) + switch op { + case "truncate": + require.NoError(t, f.mt.TruncateChannelByTime(context.Background(), segment.GetInsertChannel(), ^uint64(0))) + case "drop_channel": + require.NoError(t, f.mt.UpdateDropChannelSegmentInfo(context.Background(), segment.GetInsertChannel(), nil)) + case "batch_delete": + require.NoError(t, f.mt.DropSegmentsOfPartition(context.Background(), []int64{2})) + drainDiscovery(f.si.discovery) + n, err := f.mt.DropSegments(context.Background(), []*SegmentInfo{f.mt.GetSegment(context.Background(), 1)}) + require.NoError(t, err) + require.Equal(t, 1, n) + } + require.Equal(t, 2, discoveryPending(f.si.discovery)) + result, err := f.si.reconcileStats(statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob}, make(map[int64]statsFieldRules)) + require.NoError(t, err) + require.Equal(t, statsNotNeeded, result) + }) + } +} + +func TestStatsDiscoveryRelevantMetadataChanges(t *testing.T) { + old := discoverySegment(1, true) + cases := []struct { + name string + mutate func(*datapb.SegmentInfo) + changed bool + }{ + {"rows", func(s *datapb.SegmentInfo) { s.NumOfRows++ }, false}, + {"schema", func(s *datapb.SegmentInfo) { s.SchemaVersion++ }, true}, + {"manifest", func(s *datapb.SegmentInfo) { s.ManifestPath = "new/manifest" }, true}, + {"import", func(s *datapb.SegmentInfo) { s.IsImporting = true }, true}, + {"visibility", func(s *datapb.SegmentInfo) { s.IsInvisible = true }, true}, + {"json_stats", func(s *datapb.SegmentInfo) { s.JsonKeyStats = map[int64]*datapb.JsonKeyStats{102: {}} }, true}, + {"text_stats", func(s *datapb.SegmentInfo) { s.TextStatsLogs = map[int64]*datapb.TextIndexStats{101: {}} }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + updated := old.Clone() + tc.mutate(updated.SegmentInfo) + require.Equal(t, tc.changed, statsSegmentChanged(old, updated)) + }) + } +} diff --git a/internal/datacoord/stats_discovery_meta.go b/internal/datacoord/stats_discovery_meta.go new file mode 100644 index 00000000000..1022cfb6f12 --- /dev/null +++ b/internal/datacoord/stats_discovery_meta.go @@ -0,0 +1,76 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +// statsSegmentChanged compares only eligibility/dependency fields. In +// particular, row-count/checkpoint updates do not enqueue discovery work. +// Stats filenames need not be compared: discovery cares about field presence +// and JSON format version, not about individual generated file paths. +func statsSegmentChanged(old, current *SegmentInfo) bool { + if old == nil || current == nil { + return old != current + } + if old.GetState() != current.GetState() || old.GetLevel() != current.GetLevel() || + old.GetIsSorted() != current.GetIsSorted() || old.GetIsSortedByNamespace() != current.GetIsSortedByNamespace() || + old.GetStorageVersion() != current.GetStorageVersion() || old.GetManifestPath() != current.GetManifestPath() || + old.GetSchemaVersion() != current.GetSchemaVersion() || old.GetDataVersion() != current.GetDataVersion() || + old.GetIsImporting() != current.GetIsImporting() || old.GetIsInvisible() != current.GetIsInvisible() || + len(old.GetBinlogs()) != len(current.GetBinlogs()) { + return true + } + if len(old.GetTextStatsLogs()) != len(current.GetTextStatsLogs()) || + len(old.GetJsonKeyStats()) != len(current.GetJsonKeyStats()) { + return true + } + for id, stats := range old.GetTextStatsLogs() { + other, ok := current.GetTextStatsLogs()[id] + if !ok || (stats == nil) != (other == nil) { + return true + } + } + for id, stats := range old.GetJsonKeyStats() { + other, ok := current.GetJsonKeyStats()[id] + if !ok || (stats == nil) != (other == nil) || + stats.GetJsonKeyStatsDataFormat() != other.GetJsonKeyStatsDataFormat() { + return true + } + } + return false +} + +// Call only after successful persistence AND cache publication, never inside a +// transaction's retryable mutation callback. Consumers always reread meta; +// duplicate or stale publication notifications cannot resurrect old values. +func (m *meta) notifyStatsChange(old, current *SegmentInfo) { + q := m.statsDiscovery.Load() + if q == nil || !statsSegmentChanged(old, current) { + return + } + segment := current + if segment == nil { + segment = old + } + q.notifySegment(segment.GetCollectionID(), segment.GetID()) +} + +func (m *meta) notifyStatsSegments(collectionID int64, segmentIDs ...int64) { + if q := m.statsDiscovery.Load(); q != nil { + for _, id := range segmentIDs { + q.notifySegment(collectionID, id) + } + } +} diff --git a/internal/datacoord/stats_inspector.go b/internal/datacoord/stats_inspector.go index 9b3b399b4b2..43cb3117f35 100644 --- a/internal/datacoord/stats_inspector.go +++ b/internal/datacoord/stats_inspector.go @@ -54,7 +54,13 @@ type statsInspector struct { ctx context.Context cancel context.CancelFunc - loopWg sync.WaitGroup + loopWg sync.WaitGroup + lifecycleMu sync.Mutex // Serialize Start/Stop, including WaitGroup.Add vs Wait. + startOnce sync.Once + submitMu sync.Mutex + discovery *statsReconcileQueue + discoveryOptions statsDiscoveryOptions + discoveryUnwatch []func() mt *meta @@ -74,7 +80,7 @@ func newStatsInspector(ctx context.Context, ievm IndexEngineVersionManager, ) *statsInspector { ctx, cancel := context.WithCancel(ctx) - return &statsInspector{ + si := &statsInspector{ ctx: ctx, cancel: cancel, loopWg: sync.WaitGroup{}, @@ -85,14 +91,38 @@ func newStatsInspector(ctx context.Context, compactionInspector: compactionInspector, ievm: ievm, } + si.discoveryOptions = getStatsDiscoveryOptions() + if si.discoveryOptions.mode != "poll" { + si.discovery = newStatsReconcileQueue(si.discoveryOptions.maxPending, si.discoveryOptions.maxCollections) + mt.statsDiscovery.Store(si.discovery) + mt.statsTaskMeta.statsDiscovery.Store(si.discovery) + } + return si } func (si *statsInspector) Start() { - si.warnDeprecatedThrottleConfigs() - si.reloadFromMeta() - si.loopWg.Add(2) - go si.triggerStatsTaskLoop() - go si.cleanupStatsTasksLoop() + si.lifecycleMu.Lock() + defer si.lifecycleMu.Unlock() + if si.ctx.Err() != nil { + return + } + si.startOnce.Do(func() { + si.warnDeprecatedThrottleConfigs() + if si.discovery != nil { + si.watchStatsDiscoveryConfig() + } + si.reloadFromMeta() + si.loopWg.Add(1) + go si.cleanupStatsTasksLoop() + if si.discoveryOptions.mode != "event" { + si.loopWg.Add(1) + go si.triggerStatsTaskLoop() + } + if si.discovery != nil { + si.loopWg.Add(1) + go si.statsDiscoveryLoop() + } + }) } // warnDeprecatedThrottleConfigs tells operators whose config still carries the @@ -127,8 +157,22 @@ func jsonShreddingDisabledByDeprecatedConfig() bool { } func (si *statsInspector) Stop() { + si.lifecycleMu.Lock() + defer si.lifecycleMu.Unlock() si.cancel() + if si.discovery != nil { + si.mt.statsDiscovery.CompareAndSwap(si.discovery, nil) + si.mt.statsTaskMeta.statsDiscovery.CompareAndSwap(si.discovery, nil) + si.discovery.close() + } + for _, unwatch := range si.discoveryUnwatch { + unwatch() + } + si.discoveryUnwatch = nil si.loopWg.Wait() + if si.discovery != nil && si.mt.statsDiscovery.Load() == nil { + si.discovery.updateMetrics() + } } func (si *statsInspector) reloadFromMeta() { @@ -251,9 +295,8 @@ func needDoBM25(segment *SegmentInfo, fieldIDs []UniqueID) bool { // new stats task. The pending queue is shared by every task type, so the count is // scoped to stats work: an index or compaction backlog must not starve text-index // and JSON-shredding submission. Stats tasks waiting on a retry backoff are -// counted, because they still occupy queue depth. Discovery re-runs on every -// TaskCheckInterval tick, so a segment skipped here is picked up again once the -// stats queue drains. +// counted, because they still occupy queue depth. Poll mode checks again on its +// next tick; event mode retains refused keys in the bounded retry queue. func (si *statsInspector) canSubmitStatsTask(subJobType indexpb.StatsSubJob) bool { pendingTaskCount := si.scheduler.GetPendingTaskCount(taskcommon.Stats) pendingTaskLimit := Params.DataCoordCfg.StatsTaskPendingLimit.GetAsInt() @@ -278,9 +321,7 @@ func (si *statsInspector) triggerTextStatsTask() { } needTriggerFieldIDs := make([]UniqueID, 0) for _, field := range collection.Schema.GetFields() { - // TODO @longjiquan: please replace it to fieldSchemaHelper.EnableMath - h := typeutil.CreateFieldSchemaHelper(field) - if !h.EnableMatch() { + if !typeutil.IsMatchEnabled(field) { continue } needTriggerFieldIDs = append(needTriggerFieldIDs, field.GetFieldID()) @@ -296,9 +337,8 @@ func (si *statsInspector) triggerTextStatsTask() { return false } // A segment whose task is already in meta must not be re-submitted; - // filtering it out here keeps the per-tick work proportional to the - // segments that still need a task instead of to all of them. - // Note this runs under meta.segMu.RLock, so keep it to a map read. + // filtering it out here avoids duplicate submissions. This legacy + // selector still traverses the collection's segments. return !si.mt.statsTaskMeta.HasStatsTask(seg.GetID(), indexpb.StatsSubJob_TextIndexJob) })) @@ -342,8 +382,7 @@ func (si *statsInspector) triggerJSONKeyIndexStatsTask() { } needTriggerFieldIDs := make([]UniqueID, 0) for _, field := range collection.Schema.GetFields() { - h := typeutil.CreateFieldSchemaHelper(field) - if h.EnableJSONKeyStatsIndex() && Params.CommonCfg.EnabledJSONKeyStats.GetAsBool() { + if typeutil.IsJSONType(field.GetDataType()) && Params.CommonCfg.EnabledJSONKeyStats.GetAsBool() { needTriggerFieldIDs = append(needTriggerFieldIDs, field.GetFieldID()) } } @@ -452,9 +491,24 @@ func (si *statsInspector) SubmitStatsTask(originSegmentID, targetSegmentID int64 subJobType indexpb.StatsSubJob, canRecycle bool, resources []*internalpb.FileResourceInfo, ) error { + _, err := si.submitStatsTask(originSegmentID, targetSegmentID, subJobType, canRecycle, resources) + return err +} + +// Keep the public error-only contract, but never treat a refused admission as a +// completed event. Serializing submission also covers direct interface callers. +func (si *statsInspector) submitStatsTask(originSegmentID, targetSegmentID int64, + subJobType indexpb.StatsSubJob, canRecycle bool, + resources []*internalpb.FileResourceInfo, +) (statsSubmitResult, error) { + si.submitMu.Lock() + defer si.submitMu.Unlock() + if err := si.ctx.Err(); err != nil { + return statsDeferred, err + } originSegment := si.mt.GetHealthySegment(si.ctx, originSegmentID) if originSegment == nil { - return merr.WrapErrSegmentNotFound(originSegmentID) + return statsNotNeeded, merr.WrapErrSegmentNotFound(originSegmentID) } if si.isExternalCollection(originSegment.GetCollectionID()) { if subJobType == indexpb.StatsSubJob_JsonKeyIndexJob && !canBuildExternalJSONKeyIndex(originSegment) { @@ -462,7 +516,7 @@ func (si *statsInspector) SubmitStatsTask(originSegmentID, targetSegmentID int64 "skip submit external json stats task without v3 manifest", mlog.FieldCollectionID(originSegment.GetCollectionID()), mlog.FieldSegmentID(originSegmentID)) - return nil + return statsNotNeeded, nil } if subJobType != indexpb.StatsSubJob_TextIndexJob && subJobType != indexpb.StatsSubJob_JsonKeyIndexJob { @@ -471,7 +525,7 @@ func (si *statsInspector) SubmitStatsTask(originSegmentID, targetSegmentID int64 mlog.FieldCollectionID(originSegment.GetCollectionID()), mlog.FieldSegmentID(originSegmentID), mlog.String("subJobType", subJobType.String())) - return nil + return statsNotNeeded, nil } } if si.mt.statsTaskMeta.HasStatsTask(originSegmentID, subJobType) { @@ -479,16 +533,16 @@ func (si *statsInspector) SubmitStatsTask(originSegmentID, targetSegmentID int64 mlog.FieldCollectionID(originSegment.GetCollectionID()), mlog.FieldSegmentID(originSegmentID), mlog.String("subJobType", subJobType.String())) - return nil + return statsExisting, nil } // The trigger loops check admission before getting here; this guard covers // callers that reach the StatsInspector interface directly. if !si.canSubmitStatsTask(subJobType) { - return nil + return statsDeferred, nil } - taskID, err := si.allocator.AllocID(context.Background()) + taskID, err := si.allocator.AllocID(si.ctx) if err != nil { - return err + return statsDeferred, err } originSegmentSize := originSegment.getSegmentSize() if subJobType == indexpb.StatsSubJob_JsonKeyIndexJob { @@ -516,9 +570,9 @@ func (si *statsInspector) SubmitStatsTask(originSegmentID, targetSegmentID int64 mlog.RatedInfo(si.ctx, rate.Limit(10), "stats task already exists", mlog.FieldTaskID(taskID), mlog.FieldCollectionID(originSegment.GetCollectionID()), mlog.FieldSegmentID(originSegment.GetID())) - return nil + return statsExisting, nil } - return err + return statsDeferred, err } si.scheduler.Enqueue(newStatsTask(proto.Clone(t).(*indexpb.StatsTask), taskSlot, si.mt, si.handler, si.allocator, si.ievm)) mlog.Info(si.ctx, @@ -527,7 +581,7 @@ func (si *statsInspector) SubmitStatsTask(originSegmentID, targetSegmentID int64 mlog.FieldCollectionID(originSegment.GetCollectionID()), mlog.Int64("originSegmentID", originSegmentID), mlog.Int64("targetSegmentID", targetSegmentID), mlog.Int64("taskSlot", taskSlot)) - return nil + return statsSubmitted, nil } func (si *statsInspector) GetStatsTask(originSegmentID int64, subJobType indexpb.StatsSubJob) *indexpb.StatsTask { diff --git a/internal/datacoord/stats_reconcile.go b/internal/datacoord/stats_reconcile.go new file mode 100644 index 00000000000..8721e066b3a --- /dev/null +++ b/internal/datacoord/stats_reconcile.go @@ -0,0 +1,331 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "context" + "fmt" + "iter" + "time" + + "github.com/cockroachdb/errors" + "golang.org/x/time/rate" + + "github.com/milvus-io/milvus/internal/util/fileresource" + "github.com/milvus-io/milvus/pkg/v3/config" + "github.com/milvus-io/milvus/pkg/v3/metrics" + "github.com/milvus-io/milvus/pkg/v3/mlog" + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" + "github.com/milvus-io/milvus/pkg/v3/proto/internalpb" + "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/paramtable" + "github.com/milvus-io/milvus/pkg/v3/util/typeutil" +) + +type statsSubmitResult string + +const ( + statsSubmitted statsSubmitResult = "submitted" + statsExisting statsSubmitResult = "existing" + statsNotNeeded statsSubmitResult = "not_needed" + statsDeferred statsSubmitResult = "deferred" + statsWouldSubmit statsSubmitResult = "would_submit" +) + +type statsDiscoveryOptions struct { + mode string + maxPending, maxCollections, scanBatchSize int + scanInterval, reconcileInterval, retryInterval, retryMaxInterval time.Duration +} + +func getStatsDiscoveryOptions() statsDiscoveryOptions { + p := &Params.DataCoordCfg + return statsDiscoveryOptions{ + mode: p.StatsDiscoveryMode.GetValue(), + // Implementation budgets, not independent operator tuning knobs. + maxPending: 4096, + maxCollections: 128, + scanBatchSize: 128, + scanInterval: 100 * time.Millisecond, + reconcileInterval: p.StatsDiscoveryReconcileInterval.GetAsDuration(time.Second), + retryInterval: time.Second, + retryMaxInterval: 30 * time.Second, + } +} + +type statsScanCursor struct { + collectionID int64 + generation uint64 + next func() (int64, int64, bool) + stop func() + started time.Time + segmentID, segmentCollection int64 + subjob int + hasSegment bool +} + +// rangeStatsSegments never materializes all IDs. iter.Pull2 pauses Range at a +// yield, without holding a DataCoord metadata lock. Range is NOT a snapshot; +// concurrent insertions are covered by publication notifications/reconciliation. +func (m *meta) rangeStatsSegments(collectionID int64) iter.Seq2[int64, int64] { + return func(yield func(int64, int64) bool) { + if collectionID == 0 { + // Visit tombstones too: hiding them inside Cache.Range would allow + // one next() to walk an unbounded number of deleted entries. + m.segments.segments.entries.Range(func(id int64, entry *cacheEntry[*SegmentInfo]) bool { + if entry.deleted { + return yield(0, 0) + } + return yield(entry.value.GetCollectionID(), id) + }) + return + } + if ids, ok := m.segments.coll2Segments.Get(collectionID); ok { + ids.Range(func(id int64, _ struct{}) bool { return yield(collectionID, id) }) + } + } +} + +type statsFieldRules struct { + collection *collectionInfo + textFields, jsonFields []int64 + resources []*internalpb.FileResourceInfo + resourcesLoaded bool +} + +func (si *statsInspector) reconcileStats(key statsReconcileKey, rules map[int64]statsFieldRules) (statsSubmitResult, error) { + if err := si.ctx.Err(); err != nil { + return statsDeferred, err + } + segment := si.mt.GetHealthySegment(si.ctx, key.segmentID) + if segment == nil { + return statsNotNeeded, nil + } + collection := si.mt.GetCollection(segment.GetCollectionID()) + if collection == nil { + if si.handler == nil || si.discoveryOptions.mode == "shadow" { + return statsDeferred, nil + } + // A cache miss is not proof of deletion. Consult the existing metadata + // loader, with bounded time; only a confirmed not-found ends discovery. + ctx, cancel := context.WithTimeout(si.ctx, 10*time.Second) + var err error + collection, err = si.handler.GetCollection(ctx, segment.GetCollectionID()) + cancel() + if errors.Is(err, merr.ErrCollectionNotFound) { + return statsNotNeeded, nil + } + if err != nil || collection == nil { + return statsDeferred, err + } + } + fields, ok := rules[collection.ID] + if !ok || fields.collection != collection { + fields = statsFieldRules{collection: collection} + for _, field := range collection.Schema.GetFields() { + if typeutil.IsMatchEnabled(field) { + fields.textFields = append(fields.textFields, field.GetFieldID()) + } + if typeutil.IsJSONType(field.GetDataType()) { + fields.jsonFields = append(fields.jsonFields, field.GetFieldID()) + } + } + rules[collection.ID] = fields + } + switch key.subjob { + case indexpb.StatsSubJob_TextIndexJob: + if !needDoTextIndex(segment, fields.textFields, collection.IsExternal()) { + return statsNotNeeded, nil + } + case indexpb.StatsSubJob_JsonKeyIndexJob: + if jsonShreddingDisabledByDeprecatedConfig() || !Params.CommonCfg.EnabledJSONKeyStats.GetAsBool() || + (collection.IsExternal() && !canBuildExternalJSONKeyIndex(segment)) || + !needDoJSONKeyIndex(segment, fields.jsonFields, collection.IsExternal()) { + return statsNotNeeded, nil + } + default: + return statsNotNeeded, nil + } + if si.mt.statsTaskMeta.HasStatsTask(key.segmentID, key.subjob) { + return statsExisting, nil + } + if !si.canSubmitStatsTask(key.subjob) { + return statsDeferred, nil + } + if si.discoveryOptions.mode == "shadow" { + return statsWouldSubmit, nil // no resource RPC, ID allocation, persist or enqueue. + } + var resources []*internalpb.FileResourceInfo + if key.subjob == indexpb.StatsSubJob_TextIndexJob && + fileresource.IsRefMode(Params.CommonCfg.DNFileResourceMode.GetValue()) && + len(collection.Schema.GetFileResourceIds()) > 0 { + if !fields.resourcesLoaded { + ctx, cancel := context.WithTimeout(si.ctx, 10*time.Second) + var err error + fields.resources, err = si.mt.GetFileResources(ctx, collection.Schema.GetFileResourceIds()...) + cancel() + if err != nil { + return statsDeferred, err + } + fields.resourcesLoaded = true + rules[collection.ID] = fields + } + if si.mt.GetCollection(collection.ID) != collection { + return statsDeferred, nil // resource IDs belong to an obsolete schema. + } + resources = fields.resources + } + return si.submitStatsTask(key.segmentID, key.segmentID, key.subjob, true, resources) +} + +// advanceStatsScans shares one budget across at most four suspended iterators. +// Per-collection queue quotas and round-robin cursors stop a large collection +// from blocking smaller collection scans. A full queue retains the current ID. +func (si *statsInspector) advanceStatsScans(cursors *[]*statsScanCursor, round *int) { + q := si.discovery + for len(*cursors) < 4 { + id, gen, ok := q.beginScan() + if !ok { + break + } + next, stop := iter.Pull2(si.mt.rangeStatsSegments(id)) + *cursors = append(*cursors, &statsScanCursor{ + collectionID: id, generation: gen, next: next, stop: stop, started: time.Now(), + }) + } + jobs := [...]indexpb.StatsSubJob{indexpb.StatsSubJob_TextIndexJob, indexpb.StatsSubJob_JsonKeyIndexJob} + for budget := 0; budget < si.discoveryOptions.scanBatchSize && len(*cursors) > 0; budget++ { + if si.ctx.Err() != nil { + return + } + *round %= len(*cursors) + cursor := (*cursors)[*round] + if !cursor.hasSegment { + collectionID, segmentID, ok := cursor.next() + if !ok { + cursor.stop() + q.finishScan(cursor.collectionID, cursor.generation) + metrics.StatsDiscoveryScanDuration.Observe(time.Since(cursor.started).Seconds()) + copy((*cursors)[*round:], (*cursors)[*round+1:]) + (*cursors)[len(*cursors)-1] = nil + *cursors = (*cursors)[:len(*cursors)-1] + continue + } + metrics.StatsDiscoveryScannedSegments.Inc() + if segmentID == 0 { + *round++ + continue + } + cursor.segmentCollection, cursor.segmentID = collectionID, segmentID + cursor.subjob, cursor.hasSegment = 0, true + } + for cursor.subjob < len(jobs) { + key := statsReconcileKey{cursor.segmentID, jobs[cursor.subjob]} + if !q.enqueue(cursor.segmentCollection, key, time.Now(), false) { + break + } + cursor.subjob++ + } + cursor.hasSegment = cursor.subjob != len(jobs) + *round++ + } +} + +func (si *statsInspector) statsDiscoveryLoop() { + defer si.loopWg.Done() + opts, q := si.discoveryOptions, si.discovery + scanTicker := time.NewTicker(opts.scanInterval) + reconcileTicker := time.NewTicker(opts.reconcileInterval) + metricsTicker := time.NewTicker(time.Second) + timer := time.NewTimer(0) + defer scanTicker.Stop() + defer reconcileTicker.Stop() + defer metricsTicker.Stop() + defer timer.Stop() + var cursors []*statsScanCursor + defer func() { + for _, cursor := range cursors { + cursor.stop() + } + }() + round := 0 + q.requestScan(0, false) + for { + select { + case <-si.ctx.Done(): + return + case <-q.wake: + case <-timer.C: + case <-scanTicker.C: + si.advanceStatsScans(&cursors, &round) + case <-reconcileTicker.C: + q.requestScan(0, false) + case <-metricsTicker.C: + q.updateMetrics() + } + if si.ctx.Err() != nil { + return + } + si.processStatsDiscoveryBatch() + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(q.delay(time.Now())) + } +} + +// A soft time slice bounds local processing between select iterations. A single +// metadata/resource RPC can exceed it, but remains context-cancellable. +func (si *statsInspector) processStatsDiscoveryBatch() { + opts, q := si.discoveryOptions, si.discovery + rules := make(map[int64]statsFieldRules) + start := time.Now() + for n := 0; n < 64 && si.ctx.Err() == nil && time.Since(start) < 10*time.Millisecond; n++ { + work, ok := q.pop(time.Now()) + if !ok { + break + } + result, err := si.reconcileStats(work.key, rules) + if err != nil { + mlog.RatedWarn(si.ctx, rate.Limit(1), "stats discovery deferred after failure", + mlog.FieldSegmentID(work.key.segmentID), mlog.Err(err)) + } + metrics.StatsDiscoveryChecks.WithLabelValues(string(result), opts.mode, work.key.subjob.String()).Inc() + if result == statsSubmitted { + metrics.StatsDiscoveryDelay.Observe(time.Since(work.firstDirty).Seconds()) + } + q.complete(work, result == statsDeferred || err != nil, time.Now(), opts.retryInterval, opts.retryMaxInterval) + } +} + +func (si *statsInspector) watchStatsDiscoveryConfig() { + q := si.discovery + for _, item := range []*paramtable.ParamItem{ + &Params.CommonCfg.EnabledJSONKeyStats, &Params.DataCoordCfg.JSONStatsTriggerCount, + &Params.CommonCfg.DNFileResourceMode, + } { + keys := append([]string{item.Key}, item.FallbackKeys...) + for _, key := range keys { + handler := config.NewHandler(fmt.Sprintf("stats-discovery-%p", q), func(*config.Event) { q.requestScan(0, true) }) + Params.Watch(key, handler) + si.discoveryUnwatch = append(si.discoveryUnwatch, func() { Params.Unwatch(key, handler) }) + } + } +} diff --git a/internal/datacoord/stats_reconcile_queue.go b/internal/datacoord/stats_reconcile_queue.go new file mode 100644 index 00000000000..426d86f392e --- /dev/null +++ b/internal/datacoord/stats_reconcile_queue.go @@ -0,0 +1,330 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "container/heap" + "sync" + "time" + + "github.com/milvus-io/milvus/pkg/v3/metrics" + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" +) + +type statsReconcileKey struct { + segmentID int64 + subjob indexpb.StatsSubJob +} + +type statsPendingEntry struct { + key statsReconcileKey + collectionID int64 + generation uint64 + sequence uint64 + firstDirty time.Time + notBefore time.Time + failures uint +} + +type statsPendingHeap []*statsPendingEntry + +func (h statsPendingHeap) Len() int { return len(h) } +func (h statsPendingHeap) Less(i, j int) bool { + if h[i].notBefore.Equal(h[j].notBefore) { + return h[i].sequence < h[j].sequence + } + return h[i].notBefore.Before(h[j].notBefore) +} + +func (h statsPendingHeap) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +func (h *statsPendingHeap) Push(value any) { + e := value.(*statsPendingEntry) + *h = append(*h, e) +} + +func (h *statsPendingHeap) Pop() any { + last := len(*h) - 1 + e := (*h)[last] + (*h)[last] = nil + *h = (*h)[:last] + return e +} + +type statsScanScope struct { + generation uint64 + firstDirty time.Time +} + +// statsReconcileQueue stores dirty state, not events. Every container is bounded, +// including delayed/in-flight keys and active collection scans. Notification +// producers never wait for task persistence, worker capacity or a metadata scan. +type statsReconcileQueue struct { + mu sync.Mutex + closed bool + maxPending, maxCollections int + sequence uint64 + pending map[statsReconcileKey]*statsPendingEntry + perCollection map[int64]int + ready [2]statsPendingHeap + nextJob int + scopes map[int64]*statsScanScope // 0 is the coalesced full reconciliation. + scanReady []int64 + wake chan struct{} + overflows uint64 + reportedOverflows uint64 +} + +func statsJobSlot(job indexpb.StatsSubJob) int { + if job == indexpb.StatsSubJob_TextIndexJob { + return 0 + } + if job == indexpb.StatsSubJob_JsonKeyIndexJob { + return 1 + } + return -1 +} + +func newStatsReconcileQueue(maxPending, maxCollections int) *statsReconcileQueue { + return &statsReconcileQueue{ + maxPending: maxPending, maxCollections: maxCollections, + pending: make(map[statsReconcileKey]*statsPendingEntry), + perCollection: make(map[int64]int), + scopes: make(map[int64]*statsScanScope), + wake: make(chan struct{}, 1), + } +} + +func (q *statsReconcileQueue) signal() { + select { + case q.wake <- struct{}{}: + default: + } +} + +// enqueue with overflow=false provides scan backpressure: the scanner retains +// its current key and retries, rather than recursively requesting another scan. +func (q *statsReconcileQueue) enqueue(collectionID int64, key statsReconcileKey, now time.Time, overflow bool) bool { + slot := statsJobSlot(key.subjob) + if slot < 0 { + return true + } // Sort/BM25 are not discovered by this inspector. + q.mu.Lock() + defer q.mu.Unlock() + if q.closed { + return false + } + if e := q.pending[key]; e != nil { + // Scans are only a reconciliation request, not a new metadata version. + // Do not keep a repeatedly deferred key hot merely by scanning it again. + if overflow { + e.generation++ + } + return true + } + // Leave room for other collections during a large collection's backlog. + if len(q.pending) >= q.maxPending || q.perCollection[collectionID] >= max(1, q.maxPending/8) { + if overflow { + q.overflows++ + q.requestScanLocked(collectionID, true) + } + return false + } + q.sequence++ + e := &statsPendingEntry{ + key: key, collectionID: collectionID, generation: 1, sequence: q.sequence, + firstDirty: now, notBefore: now, + } + q.pending[key] = e + q.perCollection[collectionID]++ + heap.Push(&q.ready[slot], e) + q.signal() + return true +} + +func (q *statsReconcileQueue) notifySegment(collectionID, segmentID int64) { + now := time.Now() + q.enqueue(collectionID, statsReconcileKey{segmentID, indexpb.StatsSubJob_TextIndexJob}, now, true) + q.enqueue(collectionID, statsReconcileKey{segmentID, indexpb.StatsSubJob_JsonKeyIndexJob}, now, true) +} + +func (q *statsReconcileQueue) requestScan(collectionID int64, changed bool) { + q.mu.Lock() + defer q.mu.Unlock() + if !q.closed { + q.requestScanLocked(collectionID, changed) + } +} + +func (q *statsReconcileQueue) requestScanLocked(collectionID int64, changed bool) { + if s := q.scopes[collectionID]; s != nil { + if changed { + s.generation++ + } + return + } + collections := len(q.scopes) + if q.scopes[0] != nil { + collections-- + } + if collectionID != 0 && collections >= q.maxCollections { + q.overflows++ + q.requestScanLocked(0, changed) + return + } + q.scopes[collectionID] = &statsScanScope{generation: 1, firstDirty: time.Now()} + q.scanReady = append(q.scanReady, collectionID) +} + +func (q *statsReconcileQueue) beginScan() (collectionID int64, generation uint64, ok bool) { + q.mu.Lock() + defer q.mu.Unlock() + if q.closed || len(q.scanReady) == 0 { + return 0, 0, false + } + collectionID = q.scanReady[0] + copy(q.scanReady, q.scanReady[1:]) + q.scanReady = q.scanReady[:len(q.scanReady)-1] + s := q.scopes[collectionID] + return collectionID, s.generation, true +} + +func (q *statsReconcileQueue) finishScan(collectionID int64, generation uint64) { + q.mu.Lock() + defer q.mu.Unlock() + s := q.scopes[collectionID] + if q.closed || s == nil { + return + } + if s.generation != generation { + q.scanReady = append(q.scanReady, collectionID) + } else { + delete(q.scopes, collectionID) + } +} + +func (q *statsReconcileQueue) pop(now time.Time) (statsPendingEntry, bool) { + q.mu.Lock() + defer q.mu.Unlock() + if q.closed { + return statsPendingEntry{}, false + } + for n := 0; n < len(q.ready); n++ { + slot := (q.nextJob + n) % len(q.ready) + if len(q.ready[slot]) == 0 || q.ready[slot][0].notBefore.After(now) { + continue + } + e := heap.Pop(&q.ready[slot]).(*statsPendingEntry) + q.nextJob = (slot + 1) % len(q.ready) + return *e, true // copy generation; later notifications update the stored entry. + } + return statsPendingEntry{}, false +} + +func (q *statsReconcileQueue) complete(work statsPendingEntry, retry bool, now time.Time, retryBase, retryMax time.Duration) { + q.mu.Lock() + defer q.mu.Unlock() + e := q.pending[work.key] + if q.closed || e == nil { + return + } + if !retry && e.generation == work.generation { + delete(q.pending, work.key) + q.perCollection[e.collectionID]-- + if q.perCollection[e.collectionID] == 0 { + delete(q.perCollection, e.collectionID) + } + return + } + e.notBefore = now + if retry { + delay := min(retryBase, retryMax) + for i := uint(0); i < e.failures && delay < retryMax; i++ { + delay = min(delay*2, retryMax) + } + e.failures = min(e.failures+1, 30) + // Bounded deterministic jitter avoids an extra RNG lock on the hot path. + jitter := time.Duration(uint64(e.key.segmentID)%101) * delay / 1000 + e.notBefore = now.Add(min(delay+jitter, retryMax)) + } else { + e.failures = 0 + } + q.sequence++ + e.sequence = q.sequence + heap.Push(&q.ready[statsJobSlot(e.key.subjob)], e) +} + +func (q *statsReconcileQueue) delay(now time.Time) time.Duration { + q.mu.Lock() + defer q.mu.Unlock() + if q.closed { + return time.Hour + } + delay := time.Hour + for _, h := range q.ready { + if len(h) > 0 { + delay = min(delay, max(0, h[0].notBefore.Sub(now))) + } + } + return delay +} + +func (q *statsReconcileQueue) close() { + q.mu.Lock() + defer q.mu.Unlock() + q.closed = true + clear(q.pending) + clear(q.perCollection) + clear(q.scopes) + q.ready = [2]statsPendingHeap{} + q.scanReady = nil +} + +// Only bounded queue state is examined. Prometheus scrapes never enumerate meta. +func (q *statsReconcileQueue) updateMetrics() { + q.mu.Lock() + defer q.mu.Unlock() + metrics.StatsDiscoveryPending.WithLabelValues("key").Set(float64(len(q.pending))) + collections := len(q.scopes) + global := 0 + if q.scopes[0] != nil { + collections-- + global = 1 + } + metrics.StatsDiscoveryPending.WithLabelValues("collection").Set(float64(collections)) + metrics.StatsDiscoveryPending.WithLabelValues("global").Set(float64(global)) + var oldest time.Time + for _, e := range q.pending { + if oldest.IsZero() || e.firstDirty.Before(oldest) { + oldest = e.firstDirty + } + } + for _, s := range q.scopes { + if oldest.IsZero() || s.firstDirty.Before(oldest) { + oldest = s.firstDirty + } + } + age := float64(0) + if !oldest.IsZero() { + age = time.Since(oldest).Seconds() + } + metrics.StatsDiscoveryOldestAge.Set(age) + metrics.StatsDiscoveryOverflow.Add(float64(q.overflows - q.reportedOverflows)) + q.reportedOverflows = q.overflows +} diff --git a/internal/datacoord/stats_reconcile_queue_test.go b/internal/datacoord/stats_reconcile_queue_test.go new file mode 100644 index 00000000000..5b8dd72a17a --- /dev/null +++ b/internal/datacoord/stats_reconcile_queue_test.go @@ -0,0 +1,165 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" +) + +func TestStatsReconcileQueueGeneration(t *testing.T) { + q := newStatsReconcileQueue(16, 2) + now := time.Now() + key := statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob} + require.True(t, q.enqueue(1, key, now, true)) + for range 100 { + require.True(t, q.enqueue(1, key, now, true)) + } + require.Len(t, q.pending, 1) + work, ok := q.pop(now) + require.True(t, ok) + require.Len(t, q.pending, 1, "in-flight work still occupies capacity") + require.True(t, q.enqueue(1, key, now, true)) + q.complete(work, false, now, time.Second, time.Minute) + newer, ok := q.pop(now) + require.True(t, ok, "completion must not clear a concurrent notification") + require.Greater(t, newer.generation, work.generation) + require.Equal(t, work.firstDirty, newer.firstDirty) + q.complete(newer, false, now, time.Second, time.Minute) + require.Empty(t, q.pending) + require.Empty(t, q.perCollection) +} + +func TestStatsReconcileQueueRetryAndFairness(t *testing.T) { + q := newStatsReconcileQueue(64, 4) + now := time.Now() + for id := int64(1); id <= 3; id++ { + q.notifySegment(id, id) + } + for n := 0; n < 6; n++ { + work, ok := q.pop(time.Now()) + require.True(t, ok) + require.Equal(t, n%2, statsJobSlot(work.key.subjob), "both ready subjobs alternate") + q.complete(work, true, now, time.Second, 4*time.Second) + } + require.Len(t, q.pending, 6) + _, ok := q.pop(now) + require.False(t, ok, "a deferred key must not spin") + require.Positive(t, q.delay(now)) + later := now.Add(time.Hour) + for range 50 { + work, ok := q.pop(later) + require.True(t, ok) + require.True(t, q.enqueue(work.collectionID, work.key, later, true)) + q.complete(work, true, later, time.Second, 4*time.Second) + pending := q.pending[work.key] + require.LessOrEqual(t, pending.notBefore.Sub(later), 4*time.Second) + require.Equal(t, work.firstDirty, pending.firstDirty) + later = later.Add(time.Hour) + } +} + +func TestStatsReconcileQueueOverflowAndScanGeneration(t *testing.T) { + q := newStatsReconcileQueue(8, 1) + now := time.Now() + require.True(t, q.enqueue(1, statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob}, now, true)) + require.False(t, q.enqueue(1, statsReconcileKey{2, indexpb.StatsSubJob_TextIndexJob}, now, true)) + id, gen, ok := q.beginScan() + require.True(t, ok) + require.EqualValues(t, 1, id) + q.requestScan(1, true) + q.requestScan(2, true) // collection cap includes the active scan; promote to global. + require.Len(t, q.scopes, 2) + require.NotNil(t, q.scopes[0]) + q.finishScan(id, gen) + require.Len(t, q.scanReady, 2, "active scan finishes before the newer generation starts") + global, globalGen, ok := q.beginScan() + require.True(t, ok) + require.Zero(t, global) + q.requestScan(0, false) // periodic requests must not keep a long scan permanently dirty. + q.finishScan(global, globalGen) + require.Nil(t, q.scopes[0]) + id, gen, ok = q.beginScan() + require.True(t, ok) + q.finishScan(id, gen) + require.Empty(t, q.scopes) + for id := int64(10); id < 100; id++ { + q.enqueue(id, statsReconcileKey{id, indexpb.StatsSubJob_TextIndexJob}, now, true) + } + require.Len(t, q.pending, 8) + require.LessOrEqual(t, len(q.scopes), 2) +} + +func TestStatsReconcileQueueScanBackpressure(t *testing.T) { + q := newStatsReconcileQueue(1, 1) + now := time.Now() + key := statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob} + q.enqueue(1, key, now, true) + work, _ := q.pop(now) + require.True(t, q.enqueue(1, key, now, false)) + require.Equal(t, work.generation, q.pending[key].generation, "scanning is not a metadata change") + require.False(t, q.enqueue(1, statsReconcileKey{2, key.subjob}, now, false)) + require.Empty(t, q.scopes, "scanner backpressure must not recursively request scans") + q.complete(work, false, now, time.Second, time.Minute) + require.Empty(t, q.pending) +} + +func TestStatsReconcileQueueConcurrentClose(t *testing.T) { + q := newStatsReconcileQueue(128, 8) + var wg sync.WaitGroup + for producer := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for id := int64(1); id <= 1000; id++ { + q.notifySegment(int64(producer+1), id) + q.requestScan(int64(producer+1), true) + } + }() + } + wg.Add(1) + go func() { + defer wg.Done() + for range 1000 { + if work, ok := q.pop(time.Now()); ok { + q.complete(work, false, time.Now(), time.Second, time.Minute) + } + } + }() + q.close() + wg.Wait() + require.Empty(t, q.pending) + require.Empty(t, q.scopes) + require.False(t, q.enqueue(1, statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob}, time.Now(), true)) +} + +func BenchmarkStatsReconcileDuplicate(b *testing.B) { + q := newStatsReconcileQueue(16, 1) + key := statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob} + now := time.Now() + q.enqueue(1, key, now, true) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + q.enqueue(1, key, now, true) + } +} diff --git a/internal/datacoord/stats_reconcile_test.go b/internal/datacoord/stats_reconcile_test.go new file mode 100644 index 00000000000..c4656a9dc5f --- /dev/null +++ b/internal/datacoord/stats_reconcile_test.go @@ -0,0 +1,493 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "context" + "iter" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" + "github.com/milvus-io/milvus/internal/datacoord/allocator" + "github.com/milvus-io/milvus/internal/datacoord/task" + "github.com/milvus-io/milvus/internal/metastore" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/pkg/v3/common" + "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" + "github.com/milvus-io/milvus/pkg/v3/proto/workerpb" + "github.com/milvus-io/milvus/pkg/v3/taskcommon" + "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/paramtable" + "github.com/milvus-io/milvus/pkg/v3/util/typeutil" +) + +type discoveryTestAllocator struct { + allocator.Allocator + calls atomic.Int64 + fail atomic.Bool +} + +func (a *discoveryTestAllocator) AllocID(context.Context) (int64, error) { + id := a.calls.Add(1) + if a.fail.Load() { + return 0, merr.WrapErrServiceInternalMsg("injected allocation failure") + } + return id + 1000, nil +} + +type discoveryTestScheduler struct { + task.GlobalScheduler + pending atomic.Int64 + enqueued atomic.Int64 +} + +func (s *discoveryTestScheduler) GetPendingTaskCount(kind taskcommon.Type) int { + if kind != taskcommon.Stats { + panic("admission must be stats-scoped") + } + return int(s.pending.Load()) +} +func (s *discoveryTestScheduler) Enqueue(task.Task) { s.enqueued.Add(1) } + +type discoveryTestCatalog struct { + metastore.DataCoordCatalog + mu sync.Mutex + tasks map[int64]*indexpb.StatsTask + failSave, failDrop atomic.Bool +} + +func (c *discoveryTestCatalog) SaveStatsTask(_ context.Context, st *indexpb.StatsTask) error { + if c.failSave.Load() { + return merr.WrapErrServiceInternalMsg("injected task persistence failure") + } + c.mu.Lock() + defer c.mu.Unlock() + c.tasks[st.GetTaskID()] = proto.Clone(st).(*indexpb.StatsTask) + return nil +} + +func (c *discoveryTestCatalog) DropStatsTask(_ context.Context, id int64) error { + if c.failDrop.Load() { + return merr.WrapErrServiceInternalMsg("injected task cleanup failure") + } + c.mu.Lock() + defer c.mu.Unlock() + delete(c.tasks, id) + return nil +} + +func (c *discoveryTestCatalog) ListStatsTasks(context.Context) ([]*indexpb.StatsTask, error) { + c.mu.Lock() + defer c.mu.Unlock() + tasks := make([]*indexpb.StatsTask, 0, len(c.tasks)) + for _, st := range c.tasks { + tasks = append(tasks, proto.Clone(st).(*indexpb.StatsTask)) + } + return tasks, nil +} + +type discoveryFixture struct { + si *statsInspector + mt *meta + alloc *discoveryTestAllocator + scheduler *discoveryTestScheduler + catalog *discoveryTestCatalog +} + +func setDiscoveryTestParam(t testing.TB, item *paramtable.ParamItem, value string) { + t.Helper() + old := item.GetValue() + require.NoError(t, Params.Save(item.Key, value)) + t.Cleanup(func() { require.NoError(t, Params.Save(item.Key, old)) }) +} + +func newDiscoveryFixture(t testing.TB, mode string) *discoveryFixture { + t.Helper() + setDiscoveryTestParam(t, &Params.DataCoordCfg.StatsDiscoveryMode, mode) + setDiscoveryTestParam(t, &Params.DataCoordCfg.GCInterval, "3600") + setDiscoveryTestParam(t, &Params.DataCoordCfg.TaskCheckInterval, "3600") + setDiscoveryTestParam(t, &Params.CommonCfg.EnabledJSONKeyStats, "true") + setDiscoveryTestParam(t, &Params.DataCoordCfg.JSONStatsTriggerCount, "10") + f := &discoveryFixture{ + mt: newTestMetaWithSegments(t, NewCachedSegmentsInfo(), nil), + alloc: &discoveryTestAllocator{}, + scheduler: &discoveryTestScheduler{}, + catalog: &discoveryTestCatalog{tasks: make(map[int64]*indexpb.StatsTask)}, + } + f.mt.collections = typeutil.NewConcurrentMap[int64, *collectionInfo]() + f.mt.AddCollection(&collectionInfo{ID: 1, Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ + {FieldID: 101, DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_match", Value: "true"}}}, + {FieldID: 102, DataType: schemapb.DataType_JSON}, + }}}) + var err error + f.mt.statsTaskMeta, err = newStatsTaskMeta(context.Background(), f.catalog) + require.NoError(t, err) + f.si = newStatsInspector(context.Background(), f.mt, f.scheduler, f.alloc, nil, nil, newIndexEngineVersionManager()) + f.si.discoveryOptions.scanInterval = time.Millisecond + f.si.discoveryOptions.retryInterval = 5 * time.Millisecond + f.si.discoveryOptions.retryMaxInterval = 20 * time.Millisecond + f.si.discoveryOptions.reconcileInterval = time.Hour + t.Cleanup(f.si.Stop) + return f +} + +func discoverySegment(id int64, sorted bool) *SegmentInfo { + return NewSegmentInfo(&datapb.SegmentInfo{ + ID: id, CollectionID: 1, PartitionID: 2, InsertChannel: "stats-discovery-test", + State: commonpb.SegmentState_Flushed, Level: datapb.SegmentLevel_L1, + IsSorted: sorted, NumOfRows: 100, + }) +} + +func discoveryPending(q *statsReconcileQueue) int { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.pending) +} + +func (f *discoveryFixture) waitTasks(t *testing.T, count int) { + t.Helper() + require.Eventually(t, func() bool { + return f.mt.statsTaskMeta.tasks.Len() == count && f.scheduler.enqueued.Load() >= int64(count) + }, 5*time.Second, time.Millisecond) +} + +func drainDiscovery(q *statsReconcileQueue) { + for { + work, ok := q.pop(time.Now().Add(time.Hour)) + if !ok { + return + } + q.complete(work, false, time.Now(), time.Second, time.Minute) + } +} + +func TestStatsDiscoveryFlushSortAndDuplicate(t *testing.T) { + f := newDiscoveryFixture(t, "event") + segment := discoverySegment(1, false) + segment.State = commonpb.SegmentState_Growing + require.NoError(t, f.mt.AddSegment(context.Background(), segment)) + f.si.Start() + require.Eventually(t, func() bool { return discoveryPending(f.si.discovery) == 0 }, time.Second, time.Millisecond) + require.Zero(t, f.scheduler.enqueued.Load()) + require.NoError(t, f.mt.SetState(context.Background(), 1, commonpb.SegmentState_Flushing)) + require.Eventually(t, func() bool { return discoveryPending(f.si.discovery) == 0 }, time.Second, time.Millisecond) + require.Zero(t, f.scheduler.enqueued.Load(), "flushing but unsorted is not eligible") + require.NoError(t, f.mt.UpdateSegmentsInfo(context.Background(), map[int64][]MutateFunc{1: { + func(s *datapb.SegmentInfo) bool { s.IsSorted = true; return true }, + }})) + f.waitTasks(t, 2) + for range 100 { + f.mt.notifyStatsSegments(1, 1) + } + require.Eventually(t, func() bool { return discoveryPending(f.si.discovery) == 0 }, time.Second, time.Millisecond) + require.EqualValues(t, 2, f.alloc.calls.Load()) + require.EqualValues(t, 2, f.scheduler.enqueued.Load()) +} + +func TestStatsDiscoveryRetryWithoutNewEvent(t *testing.T) { + for _, failure := range []string{"admission", "allocation", "persistence"} { + t.Run(failure, func(t *testing.T) { + f := newDiscoveryFixture(t, "event") + switch failure { + case "admission": + f.scheduler.pending.Store(100000) + case "allocation": + f.alloc.fail.Store(true) + case "persistence": + f.catalog.failSave.Store(true) + } + require.NoError(t, f.mt.AddSegment(context.Background(), discoverySegment(1, true))) + f.si.Start() + require.Eventually(t, func() bool { + q := f.si.discovery + q.mu.Lock() + defer q.mu.Unlock() + for _, entry := range q.pending { + if entry.failures > 0 { + return true + } + } + return false + }, time.Second, time.Millisecond) + require.Zero(t, f.mt.statsTaskMeta.tasks.Len()) + require.Zero(t, f.scheduler.enqueued.Load()) + f.scheduler.pending.Store(0) + f.alloc.fail.Store(false) + f.catalog.failSave.Store(false) + // No metadata mutation, notification or periodic full scan after recovery. + f.waitTasks(t, 2) + }) + } +} + +func TestStatsDiscoveryOverflowAndLostNotification(t *testing.T) { + f := newDiscoveryFixture(t, "event") + q := newStatsReconcileQueue(8, 1) + f.si.discovery = q + f.mt.statsDiscovery.Store(q) + f.mt.statsTaskMeta.statsDiscovery.Store(q) + for id := int64(1); id <= 40; id++ { + segment := discoverySegment(id, true) + if id == 40 { + // Simulate commit/cache publication followed by crash before notification. + f.mt.statsDiscovery.Store(nil) + require.NoError(t, f.mt.AddSegment(context.Background(), segment)) + f.mt.statsDiscovery.Store(q) + } else { + require.NoError(t, f.mt.AddSegment(context.Background(), segment)) + } + } + require.Positive(t, q.overflows) + require.LessOrEqual(t, len(q.pending), 8) + f.si.Start() + f.waitTasks(t, 80) + require.Eventually(t, func() bool { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.pending) == 0 && len(q.scopes) == 0 + }, 5*time.Second, time.Millisecond, "all scan generations must converge") +} + +func TestStatsDiscoverySchemaAndConfig(t *testing.T) { + f := newDiscoveryFixture(t, "event") + setDiscoveryTestParam(t, &Params.CommonCfg.EnabledJSONKeyStats, "false") + collection := f.mt.GetClonedCollectionInfo(1) + collection.Schema.Fields = collection.Schema.Fields[1:] // JSON only + f.mt.AddCollection(collection) + require.NoError(t, f.mt.AddSegment(context.Background(), discoverySegment(1, true))) + f.si.Start() + require.Eventually(t, func() bool { return discoveryPending(f.si.discovery) == 0 }, time.Second, time.Millisecond) + require.Zero(t, f.mt.statsTaskMeta.tasks.Len()) + require.NoError(t, Params.Save(Params.CommonCfg.EnabledJSONKeyStats.Key, "true")) + f.waitTasks(t, 1) + require.True(t, f.mt.statsTaskMeta.HasStatsTask(1, indexpb.StatsSubJob_JsonKeyIndexJob)) + collection = f.mt.GetClonedCollectionInfo(1) + collection.Schema.Fields = append(collection.Schema.Fields, &schemapb.FieldSchema{ + FieldID: 103, DataType: schemapb.DataType_VarChar, + TypeParams: []*commonpb.KeyValuePair{{Key: "enable_match", Value: "true"}}, + }) + f.mt.AddCollection(collection) // no segment event; collection expansion must discover it. + f.waitTasks(t, 2) +} + +func TestStatsDiscoveryTerminalTaskCleanup(t *testing.T) { + f := newDiscoveryFixture(t, "event") + require.NoError(t, f.mt.AddSegment(context.Background(), discoverySegment(1, true))) + result, err := f.si.reconcileStats(statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob}, make(map[int64]statsFieldRules)) + require.NoError(t, err) + require.Equal(t, statsSubmitted, result) + st := f.mt.statsTaskMeta.GetStatsTaskBySegmentID(1, indexpb.StatsSubJob_TextIndexJob) + require.NoError(t, f.mt.statsTaskMeta.FinishTask(st.GetTaskID(), &workerpb.StatsResult{State: indexpb.JobState_JobStateFailed})) + result, err = f.si.reconcileStats(statsReconcileKey{1, st.GetSubJobType()}, make(map[int64]statsFieldRules)) + require.NoError(t, err) + require.Equal(t, statsExisting, result) + drainDiscovery(f.si.discovery) + f.catalog.failDrop.Store(true) + require.Error(t, f.mt.statsTaskMeta.DropStatsTask(context.Background(), st.GetTaskID())) + require.Zero(t, discoveryPending(f.si.discovery)) + require.True(t, f.mt.statsTaskMeta.HasStatsTask(1, st.GetSubJobType())) + f.catalog.failDrop.Store(false) + require.NoError(t, f.mt.statsTaskMeta.DropStatsTask(context.Background(), st.GetTaskID())) + require.Equal(t, 1, discoveryPending(f.si.discovery)) + f.si.Start() + f.waitTasks(t, 2) + require.NotEqual(t, st.GetTaskID(), f.mt.statsTaskMeta.GetStatsTaskBySegmentID(1, st.GetSubJobType()).GetTaskID()) +} + +func TestStatsDiscoveryMetadataPublication(t *testing.T) { + f := newDiscoveryFixture(t, "event") + q := f.si.discovery + segment := discoverySegment(1, false) + persist := f.mt.segmentPersist + f.mt.segmentPersist = &failingCommitSegmentPersist{base: persist, err: merr.WrapErrServiceInternalMsg("injected")} + require.Error(t, f.mt.AddSegment(context.Background(), segment)) + require.Zero(t, discoveryPending(q), "failed persistence must not notify") + f.mt.segmentPersist = persist + require.NoError(t, f.mt.AddSegment(context.Background(), segment)) + require.Equal(t, 2, discoveryPending(q)) + drainDiscovery(q) + require.NoError(t, f.mt.UpdateSegmentsInfo(context.Background(), map[int64][]MutateFunc{1: { + func(s *datapb.SegmentInfo) bool { s.NumOfRows++; return true }, + }})) + require.Zero(t, discoveryPending(q), "unrelated statistics do not cause discovery") + f.mt.segmentPersist = &failingCommitSegmentPersist{base: persist, err: merr.WrapErrServiceInternalMsg("injected")} + require.Error(t, f.mt.SetState(context.Background(), 1, commonpb.SegmentState_Dropped)) + require.Zero(t, discoveryPending(q)) + f.mt.segmentPersist = persist + require.NoError(t, f.mt.DropSegmentsOfPartition(context.Background(), []int64{2})) + require.Equal(t, 2, discoveryPending(q)) + result, err := f.si.reconcileStats(statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob}, make(map[int64]statsFieldRules)) + require.NoError(t, err) + require.Equal(t, statsNotNeeded, result) + _, version, _ := f.mt.segments.GetSegmentWithVersion(1) + require.NoError(t, f.mt.DropSegment(context.Background(), f.mt.GetSegment(context.Background(), 1))) + f.mt.segments.SetSegment(1, segment, version-1) + f.mt.notifyStatsChange(nil, segment) // late old event cannot bypass the tombstone. + result, err = f.si.reconcileStats(statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob}, make(map[int64]statsFieldRules)) + require.NoError(t, err) + require.Equal(t, statsNotNeeded, result) + require.Nil(t, f.mt.GetHealthySegment(context.Background(), 1)) +} + +func TestStatsDiscoveryShadowAndPoll(t *testing.T) { + for _, mode := range []string{"shadow", "poll"} { + t.Run(mode, func(t *testing.T) { + f := newDiscoveryFixture(t, mode) + require.NoError(t, f.mt.AddSegment(context.Background(), discoverySegment(1, true))) + if mode == "poll" { + require.Nil(t, f.si.discovery) + require.Nil(t, f.mt.statsDiscovery.Load()) + } else { + result, err := f.si.reconcileStats(statsReconcileKey{1, indexpb.StatsSubJob_TextIndexJob}, make(map[int64]statsFieldRules)) + require.NoError(t, err) + require.Equal(t, statsWouldSubmit, result) + require.Zero(t, f.alloc.calls.Load()) + require.Zero(t, f.mt.statsTaskMeta.tasks.Len()) + require.Zero(t, f.scheduler.enqueued.Load()) + } + }) + } +} + +func TestStatsDiscoveryStreamingScan(t *testing.T) { + f := newDiscoveryFixture(t, "event") + for id := int64(1); id <= 1000; id++ { + f.mt.segments.SetSegment(id, discoverySegment(id, false), 1) + } + for id := int64(1); id <= 900; id++ { + f.mt.segments.DropSegment(id, 2) + } + next, stop := iter.Pull2(f.mt.rangeStatsSegments(0)) + _, _, ok := next() + require.True(t, ok) + // A suspended iterator must not hold a metadata lock needed by writers. + done := make(chan struct{}) + go func() { f.mt.segments.SetSegment(2000, discoverySegment(2000, true), 1); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("suspended scan blocked a writer") + } + stop() + _, _, ok = next() + require.False(t, ok) + f.si.discoveryOptions.scanBatchSize = 3 + f.si.discovery.requestScan(0, false) + var cursors []*statsScanCursor + round := 0 + f.si.advanceStatsScans(&cursors, &round) + require.Len(t, cursors, 1, "a large tombstone prefix must not be skipped in one step") + require.LessOrEqual(t, discoveryPending(f.si.discovery), 6) + for _, cursor := range cursors { + cursor.stop() + } +} + +func TestStatsDiscoveryStopAndRestart(t *testing.T) { + f := newDiscoveryFixture(t, "event") + require.NoError(t, f.mt.AddSegment(context.Background(), discoverySegment(1, true))) + f.si.Start() + f.waitTasks(t, 2) + f.si.Stop() + oldQueue := f.si.discovery + require.Nil(t, f.mt.statsDiscovery.Load()) + require.Empty(t, f.si.discoveryUnwatch) + require.NoError(t, Params.Save(Params.CommonCfg.EnabledJSONKeyStats.Key, "false")) + require.Empty(t, oldQueue.scopes, "old config callbacks must be removed") + require.NoError(t, Params.Save(Params.CommonCfg.EnabledJSONKeyStats.Key, "true")) + // Reload tasks as a new coordinator would, without sharing in-memory dedup state. + reloaded, err := newStatsTaskMeta(context.Background(), f.catalog) + require.NoError(t, err) + f.mt.statsTaskMeta = reloaded + next := newStatsInspector(context.Background(), f.mt, f.scheduler, f.alloc, nil, nil, newIndexEngineVersionManager()) + next.discoveryOptions.scanInterval = time.Millisecond + t.Cleanup(next.Stop) + next.Start() + require.Eventually(t, func() bool { return f.scheduler.enqueued.Load() == 4 }, time.Second, time.Millisecond) + require.EqualValues(t, 2, f.alloc.calls.Load(), "recovery must reuse persisted task IDs") + f.si.Stop() // stopping an old inspector must not detach the replacement. + require.Same(t, next.discovery, f.mt.statsDiscovery.Load()) + var wg sync.WaitGroup + for range 4 { + wg.Add(1) + go func() { defer wg.Done(); next.Start(); next.Stop() }() + } + wg.Wait() + require.Nil(t, f.mt.statsDiscovery.Load()) +} + +func TestStatsDiscoveryEligibility(t *testing.T) { + cases := []struct { + name string + job indexpb.StatsSubJob + external bool + change func(*datapb.SegmentInfo) + want statsSubmitResult + }{ + {"text", indexpb.StatsSubJob_TextIndexJob, false, func(*datapb.SegmentInfo) {}, statsWouldSubmit}, + {"unsorted", indexpb.StatsSubJob_TextIndexJob, false, func(s *datapb.SegmentInfo) { s.IsSorted = false }, statsNotNeeded}, + {"namespace_sorted", indexpb.StatsSubJob_TextIndexJob, false, func(s *datapb.SegmentInfo) { s.IsSorted = false; s.IsSortedByNamespace = true }, statsWouldSubmit}, + {"l0", indexpb.StatsSubJob_TextIndexJob, false, func(s *datapb.SegmentInfo) { s.Level = datapb.SegmentLevel_L0 }, statsNotNeeded}, + {"dropped", indexpb.StatsSubJob_TextIndexJob, false, func(s *datapb.SegmentInfo) { s.State = commonpb.SegmentState_Dropped }, statsNotNeeded}, + {"json", indexpb.StatsSubJob_JsonKeyIndexJob, false, func(*datapb.SegmentInfo) {}, statsWouldSubmit}, + {"json_current", indexpb.StatsSubJob_JsonKeyIndexJob, false, func(s *datapb.SegmentInfo) { + s.JsonKeyStats = map[int64]*datapb.JsonKeyStats{102: {JsonKeyStatsDataFormat: common.JSONStatsDataFormatVersion}} + }, statsNotNeeded}, + {"external_text", indexpb.StatsSubJob_TextIndexJob, true, func(s *datapb.SegmentInfo) { s.IsSorted = false }, statsWouldSubmit}, + {"external_json_v2", indexpb.StatsSubJob_JsonKeyIndexJob, true, func(s *datapb.SegmentInfo) { s.StorageVersion = storage.StorageV2 }, statsNotNeeded}, + {"external_json_v3", indexpb.StatsSubJob_JsonKeyIndexJob, true, func(s *datapb.SegmentInfo) { + s.IsSorted = false + s.StorageVersion = storage.StorageV3 + s.ManifestPath = "manifest/1" + }, statsWouldSubmit}, + {"sort_disabled", indexpb.StatsSubJob_Sort, false, func(*datapb.SegmentInfo) {}, statsNotNeeded}, + {"bm25_disabled", indexpb.StatsSubJob_BM25Job, false, func(*datapb.SegmentInfo) {}, statsNotNeeded}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newDiscoveryFixture(t, "shadow") + if tc.external { + col := f.mt.GetClonedCollectionInfo(1) + col.Schema.ExternalSource = "s3://test" + for _, field := range col.Schema.Fields { + field.ExternalField = "external_column" + } + f.mt.AddCollection(col) + } + segment := discoverySegment(1, true) + tc.change(segment.SegmentInfo) + f.mt.segments.SetSegment(1, segment, 1) + got, err := f.si.reconcileStats(statsReconcileKey{1, tc.job}, make(map[int64]statsFieldRules)) + require.NoError(t, err) + require.Equal(t, tc.want, got) + if statsJobSlot(tc.job) >= 0 { + f.si.triggerStatsTasks(0) + require.Equal(t, tc.want == statsWouldSubmit, f.mt.statsTaskMeta.HasStatsTask(1, tc.job), + "legacy and event discovery must agree on the same metadata") + } + }) + } +} diff --git a/internal/datacoord/stats_task_meta.go b/internal/datacoord/stats_task_meta.go index 32af3ae48d8..8ef983756f7 100644 --- a/internal/datacoord/stats_task_meta.go +++ b/internal/datacoord/stats_task_meta.go @@ -20,6 +20,8 @@ import ( "context" "fmt" "strconv" + "sync/atomic" + "time" "golang.org/x/time/rate" "google.golang.org/protobuf/proto" @@ -45,6 +47,7 @@ type statsTaskMeta struct { // segmentID + SubJobType -> statsTask segmentID2Tasks *typeutil.ConcurrentMap[string, *indexpb.StatsTask] + statsDiscovery atomic.Pointer[statsReconcileQueue] } func newStatsTaskMeta(ctx context.Context, catalog metastore.DataCoordCatalog) (*statsTaskMeta, error) { @@ -171,6 +174,9 @@ func (stm *statsTaskMeta) DropStatsTask(ctx context.Context, taskID int64) error stm.tasks.Remove(taskID) secondaryKey := createSecondaryIndexKey(t.GetSegmentID(), t.GetSubJobType().String()) stm.segmentID2Tasks.Remove(secondaryKey) + if q := stm.statsDiscovery.Load(); q != nil { + q.enqueue(t.GetCollectionID(), statsReconcileKey{t.GetSegmentID(), t.GetSubJobType()}, time.Now(), true) + } mlog.Info(ctx, "remove stats task success", mlog.FieldTaskID(taskID)) return nil diff --git a/pkg/metrics/datacoord_metrics.go b/pkg/metrics/datacoord_metrics.go index 70e2d3f9a2d..a1871de59f0 100644 --- a/pkg/metrics/datacoord_metrics.go +++ b/pkg/metrics/datacoord_metrics.go @@ -485,6 +485,8 @@ var ( // RegisterDataCoord registers DataCoord metrics func RegisterDataCoord(registry *prometheus.Registry) { + registry.MustRegister(StatsDiscoveryPending, StatsDiscoveryOldestAge, StatsDiscoveryOverflow, + StatsDiscoveryChecks, StatsDiscoveryScannedSegments, StatsDiscoveryScanDuration, StatsDiscoveryDelay) registry.MustRegister(DataCoordNumDataNodes) registry.MustRegister(DataCoordNumSegments) registry.MustRegister(DataCoordNumCollections) diff --git a/pkg/metrics/stats_discovery_metrics.go b/pkg/metrics/stats_discovery_metrics.go new file mode 100644 index 00000000000..edce2688723 --- /dev/null +++ b/pkg/metrics/stats_discovery_metrics.go @@ -0,0 +1,57 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/milvus-io/milvus/pkg/v3/util/typeutil" +) + +// Discovery metrics have fixed, bounded labels; no segment or collection IDs. +var ( + StatsDiscoveryPending = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: milvusNamespace, Subsystem: typeutil.DataCoordRole, + Name: "stats_discovery_pending", Help: "Pending keys (including retries/in-flight), collection scopes and global scope.", + }, []string{"scope"}) + StatsDiscoveryOldestAge = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: milvusNamespace, Subsystem: typeutil.DataCoordRole, + Name: "stats_discovery_oldest_dirty_age_seconds", Help: "Age of the oldest unresolved key or reconciliation scope.", + }) + StatsDiscoveryOverflow = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: milvusNamespace, Subsystem: typeutil.DataCoordRole, + Name: "stats_discovery_overflow_total", Help: "Dirty state promotions due to segment or collection capacity limits.", + }) + StatsDiscoveryChecks = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: milvusNamespace, Subsystem: typeutil.DataCoordRole, + Name: "stats_discovery_checks_total", Help: "Stats discovery checks by result, mode and subjob.", + }, []string{"result", "mode", "subjob"}) + StatsDiscoveryScannedSegments = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: milvusNamespace, Subsystem: typeutil.DataCoordRole, + Name: "stats_discovery_scanned_entries_total", Help: "Metadata entries examined by reconciliation, including tombstones.", + }) + StatsDiscoveryScanDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: milvusNamespace, Subsystem: typeutil.DataCoordRole, + Name: "stats_discovery_scan_duration_seconds", Help: "Completed scan duration including backpressure.", + Buckets: prometheus.ExponentialBuckets(0.1, 4, 9), + }) + StatsDiscoveryDelay = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: milvusNamespace, Subsystem: typeutil.DataCoordRole, + Name: "stats_discovery_delay_seconds", Help: "Time from admitted dirty key to task submission; excludes pre-admission scan delay.", + Buckets: prometheus.ExponentialBuckets(0.01, 4, 10), + }) +) diff --git a/pkg/metrics/stats_discovery_metrics_test.go b/pkg/metrics/stats_discovery_metrics_test.go new file mode 100644 index 00000000000..da70e133623 --- /dev/null +++ b/pkg/metrics/stats_discovery_metrics_test.go @@ -0,0 +1,50 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func TestStatsDiscoveryMetricsRegistration(t *testing.T) { + registry := prometheus.NewRegistry() + RegisterDataCoord(registry) + t.Cleanup(func() { StatsDiscoveryPending.Reset(); StatsDiscoveryChecks.Reset() }) + StatsDiscoveryPending.WithLabelValues("key").Set(3) + StatsDiscoveryChecks.WithLabelValues("deferred", "event", "TextIndexJob").Inc() + require.Equal(t, float64(3), testutil.ToFloat64(StatsDiscoveryPending.WithLabelValues("key"))) + families, err := registry.Gather() + require.NoError(t, err) + count := 0 + for _, family := range families { + if !strings.HasPrefix(family.GetName(), "milvus_datacoord_stats_discovery_") { + continue + } + count++ + for _, metric := range family.Metric { + for _, label := range metric.Label { + require.Contains(t, []string{"scope", "result", "mode", "subjob"}, label.GetName()) + } + } + } + require.Equal(t, 7, count) +} diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index ba53e398b46..367651f5c26 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -5909,10 +5909,12 @@ type dataCoordConfig struct { StatsTaskSlotUsage ParamItem `refreshable:"true"` AnalyzeTaskSlotUsage ParamItem `refreshable:"true"` - EnableSortCompaction ParamItem `refreshable:"true"` - TaskCheckInterval ParamItem `refreshable:"true"` - SortCompactionTriggerCount ParamItem `refreshable:"true"` - StatsTaskPendingLimit ParamItem `refreshable:"true"` + EnableSortCompaction ParamItem `refreshable:"true"` + TaskCheckInterval ParamItem `refreshable:"true"` + SortCompactionTriggerCount ParamItem `refreshable:"true"` + StatsTaskPendingLimit ParamItem `refreshable:"true"` + StatsDiscoveryMode ParamItem `refreshable:"false"` + StatsDiscoveryReconcileInterval ParamItem `refreshable:"false"` // Deprecated: JSON stats tasks are throttled by StatsTaskPendingLimit. JSONStatsTriggerCount ParamItem `refreshable:"true"` // Deprecated: JSON stats tasks now run on TaskCheckInterval. @@ -5925,6 +5927,7 @@ type dataCoordConfig struct { } func (p *dataCoordConfig) init(base *BaseTable) { + p.initStatsDiscovery(base) p.WatchTimeoutInterval = ParamItem{ Key: "dataCoord.channel.watchTimeoutInterval", Version: "2.2.3", diff --git a/pkg/util/paramtable/stats_discovery.go b/pkg/util/paramtable/stats_discovery.go new file mode 100644 index 00000000000..bd5a8ab34a7 --- /dev/null +++ b/pkg/util/paramtable/stats_discovery.go @@ -0,0 +1,49 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paramtable + +import "strconv" + +func (p *dataCoordConfig) initStatsDiscovery(base *BaseTable) { + p.StatsDiscoveryMode = ParamItem{ + Key: "dataCoord.statsInspector.discoveryMode", Version: "3.0.0", + DefaultValue: "poll", Export: true, + Doc: "Stats task discovery: poll (legacy), shadow (read-only events), or event. Takes effect on restart.", + Formatter: func(value string) string { + switch value { + case "poll", "shadow", "event": + return value + default: + panic("dataCoord.statsInspector.discoveryMode must be poll, shadow, or event") + } + }, + } + p.StatsDiscoveryMode.Init(base.mgr) + p.StatsDiscoveryReconcileInterval = ParamItem{ + Key: "dataCoord.statsInspector.reconcileInterval", Version: "3.0.0", + DefaultValue: "600", Export: true, + Doc: "Seconds between stats reconciliation requests. Active scans are not restarted. Takes effect on restart.", + Formatter: func(value string) string { + n, err := strconv.Atoi(value) + if err != nil || n <= 0 || n > 86400 { + panic("dataCoord.statsInspector.reconcileInterval must be an integer between 1 and 86400") + } + return value + }, + } + p.StatsDiscoveryReconcileInterval.Init(base.mgr) +} diff --git a/pkg/util/paramtable/stats_discovery_test.go b/pkg/util/paramtable/stats_discovery_test.go new file mode 100644 index 00000000000..90bccbaafe9 --- /dev/null +++ b/pkg/util/paramtable/stats_discovery_test.go @@ -0,0 +1,50 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paramtable + +import ( + "reflect" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestStatsDiscoveryConfig(t *testing.T) { + base := NewBaseTable(SkipRemote(true), SkipEnv(true)) + var cfg dataCoordConfig + cfg.initStatsDiscovery(base) + require.Equal(t, "poll", cfg.StatsDiscoveryMode.GetValue()) + require.Equal(t, 10*time.Minute, cfg.StatsDiscoveryReconcileInterval.GetAsDuration(time.Second)) + for _, mode := range []string{"poll", "shadow", "event"} { + require.NoError(t, base.Save(cfg.StatsDiscoveryMode.Key, mode)) + require.Equal(t, mode, cfg.StatsDiscoveryMode.GetValue()) + } + for _, mode := range []string{"", "EVENT", "invalid"} { + require.NoError(t, base.Save(cfg.StatsDiscoveryMode.Key, mode)) + require.Panics(t, func() { cfg.StatsDiscoveryMode.GetValue() }) + } + for _, value := range []string{"0", "-1", "0.5", "invalid", "86401"} { + require.NoError(t, base.Save(cfg.StatsDiscoveryReconcileInterval.Key, value)) + require.Panics(t, func() { cfg.StatsDiscoveryReconcileInterval.GetAsInt() }) + } + for _, name := range []string{"StatsDiscoveryMode", "StatsDiscoveryReconcileInterval"} { + field, ok := reflect.TypeOf(&cfg).Elem().FieldByName(name) + require.True(t, ok) + require.Equal(t, "false", field.Tag.Get("refreshable")) + } +} diff --git a/pkg/util/typeutil/field_match_test.go b/pkg/util/typeutil/field_match_test.go new file mode 100644 index 00000000000..5c02afe5483 --- /dev/null +++ b/pkg/util/typeutil/field_match_test.go @@ -0,0 +1,66 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package typeutil + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" +) + +func TestIsMatchEnabledEquivalent(t *testing.T) { + for _, kind := range []schemapb.DataType{schemapb.DataType_VarChar, schemapb.DataType_String, schemapb.DataType_Text, schemapb.DataType_JSON, schemapb.DataType_Int64} { + for _, value := range []string{"true", "false", "TRUE", "1", "0", "bad", ""} { + for _, duplicate := range []bool{false, true} { + field := &schemapb.FieldSchema{DataType: kind, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_match", Value: value}}} + if duplicate { + field.TypeParams = append(field.TypeParams, &commonpb.KeyValuePair{Key: "enable_match", Value: "false"}) + } + require.Equal(t, CreateFieldSchemaHelper(field).EnableMatch(), IsMatchEnabled(field)) + } + } + field := &schemapb.FieldSchema{DataType: kind} + require.Equal(t, CreateFieldSchemaHelper(field).EnableMatch(), IsMatchEnabled(field)) + } + require.False(t, IsMatchEnabled(nil)) +} + +func TestIsMatchEnabledNoAllocation(t *testing.T) { + for _, params := range [][]*commonpb.KeyValuePair{nil, {{Key: "enable_match", Value: "true"}}} { + field := &schemapb.FieldSchema{DataType: schemapb.DataType_VarChar, TypeParams: params} + require.Zero(t, testing.AllocsPerRun(100, func() { IsMatchEnabled(field) })) + } +} + +func BenchmarkMatchFieldCheck(b *testing.B) { + field := &schemapb.FieldSchema{DataType: schemapb.DataType_VarChar} + b.Run("legacy_helper", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + CreateFieldSchemaHelper(field).EnableMatch() + } + }) + b.Run("direct", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + IsMatchEnabled(field) + } + }) +} diff --git a/pkg/util/typeutil/field_schema.go b/pkg/util/typeutil/field_schema.go index 245c7f1273b..7528efd6e4e 100644 --- a/pkg/util/typeutil/field_schema.go +++ b/pkg/util/typeutil/field_schema.go @@ -54,6 +54,22 @@ func (h *FieldSchemaHelper) EnableMatch() bool { return err == nil && enable } +// IsMatchEnabled checks type parameters without allocating a FieldSchemaHelper. +// Like NewKvPairs, the last value wins when a key appears more than once. +func IsMatchEnabled(field *schemapb.FieldSchema) bool { + if !IsStringType(field.GetDataType()) { + return false + } + params := field.GetTypeParams() + for i := len(params) - 1; i >= 0; i-- { + if params[i].GetKey() == "enable_match" { + enabled, err := strconv.ParseBool(params[i].GetValue()) + return err == nil && enabled + } + } + return false +} + func (h *FieldSchemaHelper) EnableJSONKeyStatsIndex() bool { return IsJSONType(h.schema.GetDataType()) }