From ac7acaf61bfaf017a427166db846db81c6a0e629 Mon Sep 17 00:00:00 2001 From: Amber Date: Mon, 10 Aug 2026 13:57:49 +0800 Subject: [PATCH 01/16] feat: add HugeGraph standalone addon --- .github/CODEOWNERS | 3 + .../release-addons-cluster-chart.yml | 1 + .github/workflows/release-chart.yml | 1 + README.md | 1 + addons-cluster/hugegraph/.helmignore | 5 + addons-cluster/hugegraph/Chart.yaml | 23 ++ addons-cluster/hugegraph/releases_notes.yaml | 8 + addons-cluster/hugegraph/templates/NOTES.txt | 5 + .../hugegraph/templates/cluster.yaml | 16 + addons-cluster/hugegraph/values.schema.json | 50 +++ addons-cluster/hugegraph/values.yaml | 13 + addons/hugegraph/.helmignore | 6 + addons/hugegraph/Chart.yaml | 28 ++ addons/hugegraph/DESIGN.zh.md | 170 ++++++++++ addons/hugegraph/README.md | 69 ++++ addons/hugegraph/releases_notes.yaml | 8 + addons/hugegraph/scripts/backup.sh | 231 +++++++++++++ addons/hugegraph/scripts/restore.sh | 321 ++++++++++++++++++ addons/hugegraph/scripts/start.sh | 130 +++++++ addons/hugegraph/templates/_helpers.tpl | 53 +++ addons/hugegraph/templates/actionset.yaml | 34 ++ .../templates/backuppolicytemplate.yaml | 29 ++ .../templates/clusterdefinition.yaml | 15 + addons/hugegraph/templates/cmpd.yaml | 120 +++++++ addons/hugegraph/templates/cmpv.yaml | 19 ++ .../hugegraph/templates/script-template.yaml | 9 + addons/hugegraph/tests/contract_test.sh | 107 ++++++ addons/hugegraph/tests/scripts_test.sh | 208 ++++++++++++ addons/hugegraph/values.yaml | 14 + examples/hugegraph/README.md | 37 ++ examples/hugegraph/backup.yaml | 9 + examples/hugegraph/cluster.yaml | 29 ++ examples/hugegraph/restart.yaml | 10 + examples/hugegraph/restore.yaml | 31 ++ examples/hugegraph/start.yaml | 10 + examples/hugegraph/stop.yaml | 10 + examples/hugegraph/verticalscale.yaml | 16 + examples/hugegraph/volumeexpand.yaml | 13 + 38 files changed, 1862 insertions(+) create mode 100644 addons-cluster/hugegraph/.helmignore create mode 100644 addons-cluster/hugegraph/Chart.yaml create mode 100644 addons-cluster/hugegraph/releases_notes.yaml create mode 100644 addons-cluster/hugegraph/templates/NOTES.txt create mode 100644 addons-cluster/hugegraph/templates/cluster.yaml create mode 100644 addons-cluster/hugegraph/values.schema.json create mode 100644 addons-cluster/hugegraph/values.yaml create mode 100644 addons/hugegraph/.helmignore create mode 100644 addons/hugegraph/Chart.yaml create mode 100644 addons/hugegraph/DESIGN.zh.md create mode 100644 addons/hugegraph/README.md create mode 100644 addons/hugegraph/releases_notes.yaml create mode 100644 addons/hugegraph/scripts/backup.sh create mode 100644 addons/hugegraph/scripts/restore.sh create mode 100644 addons/hugegraph/scripts/start.sh create mode 100644 addons/hugegraph/templates/_helpers.tpl create mode 100644 addons/hugegraph/templates/actionset.yaml create mode 100644 addons/hugegraph/templates/backuppolicytemplate.yaml create mode 100644 addons/hugegraph/templates/clusterdefinition.yaml create mode 100644 addons/hugegraph/templates/cmpd.yaml create mode 100644 addons/hugegraph/templates/cmpv.yaml create mode 100644 addons/hugegraph/templates/script-template.yaml create mode 100644 addons/hugegraph/tests/contract_test.sh create mode 100755 addons/hugegraph/tests/scripts_test.sh create mode 100644 addons/hugegraph/values.yaml create mode 100644 examples/hugegraph/README.md create mode 100644 examples/hugegraph/backup.yaml create mode 100644 examples/hugegraph/cluster.yaml create mode 100644 examples/hugegraph/restart.yaml create mode 100644 examples/hugegraph/restore.yaml create mode 100644 examples/hugegraph/start.yaml create mode 100644 examples/hugegraph/stop.yaml create mode 100644 examples/hugegraph/verticalscale.yaml create mode 100644 examples/hugegraph/volumeexpand.yaml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 276cb91f80..67dd2b706c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -22,6 +22,9 @@ addons-cluster/etcd/ @loomt @leon-ape @apecloud/kb-reviewers @apecloud/kb-addon- addons/greptimedb/ @GreptimeTeam @sh2 @leon-ape @apecloud/kb-reviewers @apecloud/kb-addon-reviewers addons-cluster/greptimedb/ @GreptimeTeam @sh2 @leon-ape @apecloud/kb-reviewers @apecloud/kb-addon-reviewers +addons/hugegraph/ @leon-ape @apecloud/kb-reviewers @apecloud/kb-addon-reviewers +addons-cluster/hugegraph/ @leon-ape @apecloud/kb-reviewers @apecloud/kb-addon-reviewers + addons/influxdb/ @cjc7373 @leon-ape @apecloud/kb-reviewers @apecloud/kb-addon-reviewers addons-cluster/influxdb/ @cjc7373 @leon-ape @apecloud/kb-reviewers @apecloud/kb-addon-reviewers diff --git a/.github/workflows/release-addons-cluster-chart.yml b/.github/workflows/release-addons-cluster-chart.yml index dac5c14295..7e78846e09 100644 --- a/.github/workflows/release-addons-cluster-chart.yml +++ b/.github/workflows/release-addons-cluster-chart.yml @@ -16,6 +16,7 @@ on: - elasticsearch - etcd - greptimedb + - hugegraph - influxdb - kafka - llm diff --git a/.github/workflows/release-chart.yml b/.github/workflows/release-chart.yml index 0901adeb72..5376c577ee 100644 --- a/.github/workflows/release-chart.yml +++ b/.github/workflows/release-chart.yml @@ -16,6 +16,7 @@ on: - elasticsearch - etcd - greptimedb + - hugegraph - influxdb - kafka - llm diff --git a/README.md b/README.md index 4c3fb6fcb7..15f7291234 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ KubeBlocks add-ons. | elasticsearch | elasticsearch-6.8.23
elasticsearch-7.10.1
elasticsearch-7.10.2
elasticsearch-7.7.1
elasticsearch-7.8.1
elasticsearch-8.1.3
elasticsearch-8.15.5
elasticsearch-8.8.2
kibana-6.8.23
kibana-7.10.1
kibana-7.10.2
kibana-7.7.1
kibana-7.8.1
kibana-8.1.3
kibana-8.15.5
kibana-8.8.2
kibana-8.9.1 | Elasticsearch is a distributed, RESTful search engine optimized for speed and relevance on production-scale workloads. | kizuna-lek vipshop | | etcd | etcd-3.5.15
etcd-3.5.6
etcd-3.6.1 | Etcd is a strongly consistent, distributed key-value store that provides a reliable way to store data that needs to be accessed by a distributed system or cluster of machines. | loomt | | greptimedb | greptimedb-0.3.2 | An open-source, cloud-native, distributed time-series database with PromQL/SQL/Python supported. | GreptimeTeam sh2 | +| hugegraph | hugegraph-1.7.0 | Apache HugeGraph standalone addon for KubeBlocks. | ApeCloud | | influxdb | influxdb-2.7.11
influxdb-data-1.8.10
influxdb-meta-1.8.10 | InfluxDB is an open source time-series database. It is a core component of the TICK (Telegraf, InfluxDB(TM), Chronograf, Kapacitor) stack. | cjc7373 | | kafka | kafka-broker-2.7.0
kafka-broker-2.8.2
kafka-broker-3.3.2
kafka-broker-3.7.1
kafka-broker-3.8.1
kafka-broker-3.9.0
kafka-combine-3.3.2
kafka-combine-3.7.1
kafka-combine-3.8.1
kafka-combine-3.9.0
kafka-controller-3.3.2
kafka-controller-3.7.1
kafka-controller-3.8.1
kafka-controller-3.9.0
kafka-exporter-1.6.0 | Apache Kafka is a distributed streaming platform designed to build real-time pipelines and can be used as a message broker or as a replacement for a log aggregation solution for big data applications. | cjc7373 caiq1nyu vipshop | | llm | llm-ggml-20241115-latest
llm-vllm-0.11.2 | Large language models. | ApeCloud | diff --git a/addons-cluster/hugegraph/.helmignore b/addons-cluster/hugegraph/.helmignore new file mode 100644 index 0000000000..bf60abe5f7 --- /dev/null +++ b/addons-cluster/hugegraph/.helmignore @@ -0,0 +1,5 @@ +.DS_Store +.git/ +.gitignore +*.swp +*.tmp diff --git a/addons-cluster/hugegraph/Chart.yaml b/addons-cluster/hugegraph/Chart.yaml new file mode 100644 index 0000000000..b2f9232c28 --- /dev/null +++ b/addons-cluster/hugegraph/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: hugegraph-cluster +description: Apache HugeGraph standalone cluster chart for KubeBlocks. +type: application +version: 1.0.0 +appVersion: "1.7.0" + +dependencies: + - name: kblib + version: 0.1.2 + repository: file://../kblib + alias: extra + +home: https://hugegraph.apache.org/ +icon: https://hugegraph.apache.org/img/logo.svg + +maintainers: + - name: ApeCloud + url: https://kubeblocks.io/ + +sources: + - https://github.com/apache/incubator-hugegraph + - https://github.com/apecloud/kubeblocks-addons diff --git a/addons-cluster/hugegraph/releases_notes.yaml b/addons-cluster/hugegraph/releases_notes.yaml new file mode 100644 index 0000000000..1eb2e8c4d5 --- /dev/null +++ b/addons-cluster/hugegraph/releases_notes.yaml @@ -0,0 +1,8 @@ +releases: + - version: "1.0.0" + released_at: "2026-08-10" + status: preview + notes: Initial HugeGraph 1.7.0 standalone cluster chart. + git_branch: "release-1.0" + git_tag: "1.0.0" + commit_sha: "" diff --git a/addons-cluster/hugegraph/templates/NOTES.txt b/addons-cluster/hugegraph/templates/NOTES.txt new file mode 100644 index 0000000000..a2b6689d54 --- /dev/null +++ b/addons-cluster/hugegraph/templates/NOTES.txt @@ -0,0 +1,5 @@ +HugeGraph {{ .Values.version }} standalone cluster {{ include "kblib.clusterName" . }} has been submitted. + +HTTP service port: 8080 +Gremlin service port: 8182 +System account Secret: {{ include "kblib.clusterName" . }}-server-account-admin diff --git a/addons-cluster/hugegraph/templates/cluster.yaml b/addons-cluster/hugegraph/templates/cluster.yaml new file mode 100644 index 0000000000..ee27e18518 --- /dev/null +++ b/addons-cluster/hugegraph/templates/cluster.yaml @@ -0,0 +1,16 @@ +apiVersion: apps.kubeblocks.io/v1 +kind: Cluster +metadata: + name: {{ include "kblib.clusterName" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "kblib.clusterLabels" . | nindent 4 }} +spec: + clusterDef: hugegraph + topology: standalone + terminationPolicy: {{ .Values.extra.terminationPolicy }} + componentSpecs: + - name: server + serviceVersion: {{ .Values.version | quote }} + replicas: 1 + {{- include "kblib.componentResources" . | nindent 6 }} + {{- include "kblib.componentStorages" . | nindent 6 }} diff --git a/addons-cluster/hugegraph/values.schema.json b/addons-cluster/hugegraph/values.schema.json new file mode 100644 index 0000000000..6a7f9b8c21 --- /dev/null +++ b/addons-cluster/hugegraph/values.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "version": { + "title": "Version", + "description": "HugeGraph service version.", + "type": "string", + "default": "1.7.0", + "enum": ["1.7.0"] + }, + "replicas": { + "title": "Replicas", + "description": "Standalone HugeGraph requires exactly one replica.", + "type": "integer", + "default": 1, + "minimum": 1, + "maximum": 1 + }, + "cpu": { + "title": "CPU", + "description": "CPU cores.", + "type": ["number", "string"], + "default": 1, + "minimum": 0.5, + "maximum": 64 + }, + "memory": { + "title": "Memory(Gi)", + "description": "Memory in Gi.", + "type": ["number", "string"], + "default": 2, + "minimum": 1, + "maximum": 1000 + }, + "storage": { + "title": "Storage(Gi)", + "description": "Data volume size in Gi.", + "type": ["number", "string"], + "default": 20, + "minimum": 1, + "maximum": 10000 + }, + "storageClassName": { + "title": "Storage Class Name", + "description": "StorageClass for the data PVC.", + "type": "string" + } + } +} diff --git a/addons-cluster/hugegraph/values.yaml b/addons-cluster/hugegraph/values.yaml new file mode 100644 index 0000000000..b8f5ca9fc6 --- /dev/null +++ b/addons-cluster/hugegraph/values.yaml @@ -0,0 +1,13 @@ +version: "1.7.0" +replicas: 1 + +cpu: 1 +memory: 2 + +requests: + +storage: 20 +storageClassName: "" + +extra: + terminationPolicy: Delete diff --git a/addons/hugegraph/.helmignore b/addons/hugegraph/.helmignore new file mode 100644 index 0000000000..867c3249d6 --- /dev/null +++ b/addons/hugegraph/.helmignore @@ -0,0 +1,6 @@ +.DS_Store +.git/ +.gitignore +tests/ +*.swp +*.tmp diff --git a/addons/hugegraph/Chart.yaml b/addons/hugegraph/Chart.yaml new file mode 100644 index 0000000000..57fcf2ecf7 --- /dev/null +++ b/addons/hugegraph/Chart.yaml @@ -0,0 +1,28 @@ +apiVersion: v2 +name: hugegraph +description: Apache HugeGraph standalone addon for KubeBlocks. +type: application +version: 1.0.0 +appVersion: "1.7.0" + +dependencies: + - name: kblib + version: 0.1.0 + repository: file://../kblib + alias: extra + +home: https://hugegraph.apache.org/ +icon: https://hugegraph.apache.org/img/logo.svg + +maintainers: + - name: ApeCloud + url: https://kubeblocks.io/ + +sources: + - https://github.com/apache/incubator-hugegraph + - https://github.com/apecloud/kubeblocks-addons + +annotations: + addon.kubeblocks.io/kubeblocks-version: ">=1.0.0" + addon.kubeblocks.io/model: "graph" + addon.kubeblocks.io/provider: "community" diff --git a/addons/hugegraph/DESIGN.zh.md b/addons/hugegraph/DESIGN.zh.md new file mode 100644 index 0000000000..2f156dd4cf --- /dev/null +++ b/addons/hugegraph/DESIGN.zh.md @@ -0,0 +1,170 @@ +# HugeGraph 1.7.0 单节点 Addon 设计 + +## 1. 问题与证据 + +目标是在 KubeBlocks Addons `release-1.0` 上提供 HugeGraph 1.7.0 单节点 +RocksDB addon,并支持整实例备份恢复。首版不能依赖 CSI VolumeSnapshot。 + +HugeGraph 1.7.0 提供 graph 级 RocksDB checkpoint API: + +- `PUT /graphspaces/DEFAULT/graphs/{graph}/snapshot_create` +- `PUT /graphspaces/DEFAULT/graphs/{graph}/snapshot_resume` + +`snapshot_create` 会为 graph 的 schema、system、graph 三类 RocksDB store 创建 +checkpoint。checkpoint 位于原始 RocksDB 目录的同级 `snapshot_*` 目录。该行为在 +HugeGraph 1.7.0 的 `GraphsAPI`、`AbstractBackendStoreProvider`、`RocksDBStore` 和 +`RocksDBStdSessions` 中可追溯。 + +KubeBlocks `release-1.0` 的非 snapshot 数据保护链路通过 +`BackupPolicyTemplate + ActionSet` 工作。`prepareData` Job 会把目标 PVC 挂载到 +Backup method 声明的 mountPath,因此可以在目标 Pod 启动前把 checkpoint 写回新 +PVC。 + +## 2. 范围 + +首版支持: + +- HugeGraph Server 1.7.0,单副本,内嵌 RocksDB。 +- HTTP 8080 和 Gremlin 8182 服务。 +- `admin` init system account;探针和数据保护均使用该账号。 +- 一个 `data` PVC,保存 graph 配置、RocksDB、WAL 和初始化标记。 +- 对实例内所有持久化 graph 做 full checkpoint backup。 +- 从 full checkpoint backup 恢复到同版本、同 topology 的新 Cluster。 +- Restart、Stop/Start、VerticalScaling、VolumeExpansion 和 Expose 的底层合同。 + +首版不支持: + +- CSI volume snapshot、PITR、增量备份、单 graph 选择性恢复。 +- 多 graph 之间的全局事务时间点。每个 graph 自身 checkpoint 一致,但 graph 之间 + 按顺序创建 checkpoint,存在短时间窗口。 +- RebuildInstance、scaleOut.fromBackup、跨 HugeGraph 版本、跨 topology 恢复。 +- TLS、在线参数变更、分布式 PD/Store/Server topology。 + +## 3. 存储和启动 + +PVC 固定挂载到 `/hugegraph-data`: + +```text +/hugegraph-data/ + graphs/ # 持久化 graph properties + rocksdb/ # 默认 hugegraph 数据 + rocksdb_/ # clone graph 数据 + wal/ # 默认 hugegraph WAL + wal_/ # clone graph WAL + docker/init_complete # 上游 entrypoint 初始化标记 +``` + +不把 PVC 覆盖到 `/hugegraph-server`,避免遮蔽镜像内二进制、插件和默认配置。 +启动脚本执行以下动作后调用上游 `docker-entrypoint.sh`: + +1. 首次启动时把默认 `hugegraph.properties` 复制到持久化 `graphs/`。 +2. 把 `rest-server.properties` 的 `graphs` 指向持久化目录并监听 `0.0.0.0`。 +3. 校验每个 graph 都使用 RocksDB,且 data/WAL 路径是 PVC 根目录的直接子目录。 +4. 为每个 graph 启用 `HugeFactoryAuthProxy`。 +5. 把上游 `docker/` 初始化标记目录链接到 PVC。 + +动态多图推荐使用 `clone_graph_name=hugegraph` 创建。HugeGraph 的 RocksDB provider +会为 clone graph 自动生成独立的 `rocksdb_` 和 `wal_` 路径。直接 +创建 graph 时必须显式提供满足上述路径约束的 `rocksdb.data_path` 和 +`rocksdb.wal_path`;否则备份前置检查会失败,不会静默漏备份。 + +## 4. 备份流程 + +Backup method 名为 `checkpoint`,`snapshotVolumes=false`。ActionSet 使用 +`hugegraph/hugegraph:1.7.0`,不增加自建工具镜像。 + +1. 在 PVC 上获取原子目录锁,拒绝并发 checkpoint backup。 +2. 枚举 `/hugegraph-data/graphs/*.properties`,至少要求一个 graph。 +3. 校验 backend、store、RocksDB data/WAL 路径和认证代理。 +4. 对每个 graph 调用 `snapshot_create`。 +5. 枚举、限制并验证所有 `snapshot_*` 目录非空。 +6. 生成 `manifest.properties` 和 `checksums.sha256`。 +7. 上传 manifest、checksum 和 `payload.tar.gz` 到当前 datasafed backup path。 +8. 更新 backup size;无论成功失败,只清理由本次流程记录的 checkpoint 和锁。 + +artifact format v1: + +```text +manifest.properties +checksums.sha256 +payload.tar.gz + graphs/*.properties + snapshot_*/... + manifest.properties + checksums.sha256 +``` + +manifest 记录 format、engine/service version、graph 名和配置 hash、checkpoint source +和目标 RocksDB 目录。恢复必须拒绝未知 format、版本不一致、空 graph/checkpoint、 +checksum 错误和越界路径。 + +## 5. 恢复流程 + +恢复在 `prepareData` 执行,不等待业务 Pod Ready: + +1. 一次性拉取外层 manifest、checksum 和 payload,后续校验与解压使用同一份文件。 +2. 先只列出 payload,拒绝绝对路径、`..`、链接和非白名单成员。 +3. 流式解压到目标 PVC 内的隔离 staging 目录。 +4. 比较内外 manifest/checksum,并执行 `sha256sum -c`。 +5. 根据 graph config 验证 checkpoint 与 RocksDB data 目录一一对应,再把 + `snapshot_` 原子移动为 ``,并移动 `graphs/`。 +6. 创建持久化 `docker/init_complete`,确保上游 entrypoint 不会对已恢复 RocksDB + 再执行初始化。 +7. 先同步数据,再写入本次 backup 的完成标记并删除 staging。 + +恢复不在 `postReady` 调用 `snapshot_resume`。原因是新 Cluster 的 PVC 本来为空, +checkpoint 可以在启动前直接成为 RocksDB 原目录;这样 HugeGraph 第一次打开的就是 +恢复数据,避免先以空库 Ready、再在线替换 auth/store 所产生的可见窗口和缓存风险。 +`snapshot_resume` 保留为 HugeGraph 在线回退 API,不是首版新 Cluster restore 阶段。 + +prepareData 通过“进行中标记 + backup name”处理重试。只有标记属于同一个 backup 时 +才允许清理由该次失败留下的目标目录;不删除未标记的现有业务目录。完成态重试会重新 +校验每个 graph config 和已恢复 checkpoint 文件的 SHA-256,不只检查目录存在。 + +## 6. 账号兼容 + +默认 graph 的 auth 数据包含在 checkpoint 中。KubeBlocks `release-1.0` 会把 full +Backup 的加密 system-account metadata 传递给 restore,并重建目标 init-account +Secret,因此目标 `admin` Secret 应与源 backup 对应。运行验证必须同时检查 Secret +和实际 HTTP 登录;在获得 live evidence 前,这一点只作为版本锁定的控制器能力, +不扩展为跨版本保证。 + +## 7. 兼容性矩阵 + +| 场景 | 首版结论 | +| --- | --- | +| 新 Cluster,同 topology,1.7.0 -> 1.7.0 | 支持,需真实 backup/restore 验证 | +| 空 graph | 支持;checkpoint 仍必须包含 RocksDB 元数据文件 | +| 多 graph | 支持逐 graph checkpoint;不保证跨 graph 同一时间点 | +| TLS on/off | TLS 不声明支持 | +| RebuildInstance / scaleOut.fromBackup | 不支持 | +| 跨 serviceVersion / topology | 不支持,restore preflight 拒绝 | +| PITR / incremental / selective | 不支持 | + +## 8. 测试计划 + +开发侧离线合同测试: + +- `helm lint` 和 definition/cluster chart 的代表性 `helm template`。 +- 名称、compDef、ActionSet、BPT、volume、service port、system account 引用闭合。 +- 明确断言不存在 `volume-snapshot` 或 `snapshotVolumes: true`。 +- Bash 语法、manifest/checksum、路径白名单、checkpoint 清理和恢复重试合同。 +- restore example 固定单副本、1.7.0、目标 volume 与 source target 映射。 + +Tester 侧 live 验证由固定 HugeGraph tester 独立组包和执行,至少覆盖:单 graph、空 +graph、多 graph、并发备份拒绝、checkpoint API 失败、artifact 缺失/checksum 损坏、 +新 Cluster restore、恢复后读写、admin Secret/实际登录一致、Restart、Stop/Start、 +VerticalScaling 和 VolumeExpansion。 + +## 9. 合同来源 + +- `kubeblocks-addon-docs/docs/addon-api/02-component-definition.md` +- `kubeblocks-addon-docs/docs/addon-api/03-cluster-definition.md` +- `kubeblocks-addon-docs/docs/addon-api/04-component-version.md` +- `kubeblocks-addon-docs/docs/addon-api/07-accounts-and-tls.md` +- `kubeblocks-addon-docs/docs/addon-api/09a-backup-basic.md` +- `kubeblocks-addon-docs/docs/addon-api/09b-backup-extensions.md` +- `kubeblocks-addon-docs/docs/addon-api/09c-restore-and-rebuild.md` +- `kubeblocks-addon-docs/docs/addon-api/10-day2-operations.md` +- `kubeblocks-addon-docs/docs/addon-api/12a-minimum-acceptance.md` +- `kubeblocks-addon-docs/docs/addon-api/12b-claimed-only-acceptance.md` diff --git a/addons/hugegraph/README.md b/addons/hugegraph/README.md new file mode 100644 index 0000000000..247556f606 --- /dev/null +++ b/addons/hugegraph/README.md @@ -0,0 +1,69 @@ +# HugeGraph + +This addon runs Apache HugeGraph 1.7.0 as a single HugeGraph Server backed by +RocksDB. + +## Capabilities + +| Capability | Standalone | +| --- | --- | +| Replicas | Exactly 1 | +| Persistent data | Yes | +| Restart | Yes | +| Stop/Start | Yes | +| Vertical scaling | Yes | +| Volume expansion | Yes, when the StorageClass supports expansion | +| Expose | Yes | +| Full backup/restore | RocksDB checkpoint | +| Horizontal scaling | No | +| Reconfigure | No | +| TLS | No | + +The `checkpoint` backup method calls HugeGraph's `snapshot_create` API for +every persistent graph. It uploads graph configurations, a format-versioned +manifest, SHA-256 checksums, and all RocksDB checkpoints through datasafed. It +does not use CSI volume snapshots. + +Restore is supported only to a new HugeGraph 1.7.0 standalone Cluster. It +restores the complete instance. Single-graph restore, PITR, incremental backup, +RebuildInstance, cross-version restore, and cross-topology restore are not +supported. The restore action validates every graph config and checkpoint file, +then creates the persistent upstream initialization marker before the first +HugeGraph process starts. + +## Storage + +The `data` PVC is mounted at `/hugegraph-data`. The addon keeps graph +configuration files, RocksDB directories, WAL directories, and the upstream +initialization marker on this volume. It does not mount a PVC over +`/hugegraph-server`. + +For additional graphs, use the HugeGraph clone API so the RocksDB provider +creates unique persistent paths: + +```bash +curl --fail --user 'admin:' \ + --request POST \ + 'http://:8080/graphspaces/DEFAULT/graphs/analytics?clone_graph_name=hugegraph' \ + --header 'Content-Type: application/json' \ + --data '{}' +``` + +When creating a graph without `clone_graph_name`, explicitly set unique +`rocksdb.data_path` and `rocksdb.wal_path` values that are direct children of +`/hugegraph-data`. Backup fails rather than silently omitting a graph whose +data is outside the managed PVC. + +Each graph checkpoint is internally consistent. Checkpoints are created one +graph at a time, so this method does not provide a single cross-graph +transaction timestamp. + +## Image + +The runtime and data protection actions use +`docker.io/hugegraph/hugegraph:1.7.0`. No custom tools image is required. + +## Examples + +See `examples/hugegraph` for Cluster, backup, restore, restart, stop/start, +vertical scaling, and volume expansion manifests. diff --git a/addons/hugegraph/releases_notes.yaml b/addons/hugegraph/releases_notes.yaml new file mode 100644 index 0000000000..8b0cc49e75 --- /dev/null +++ b/addons/hugegraph/releases_notes.yaml @@ -0,0 +1,8 @@ +releases: + - version: "1.0.0" + released_at: "2026-08-10" + status: preview + notes: Initial HugeGraph 1.7.0 standalone release with RocksDB checkpoint backup and restore. + git_branch: "release-1.0" + git_tag: "1.0.0" + commit_sha: "" diff --git a/addons/hugegraph/scripts/backup.sh b/addons/hugegraph/scripts/backup.sh new file mode 100644 index 0000000000..0f9becf961 --- /dev/null +++ b/addons/hugegraph/scripts/backup.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +DATA_ROOT=${DATA_ROOT:-/hugegraph-data} +GRAPH_DIR="${DATA_ROOT}/graphs" +API_PORT=${API_PORT:-8080} +FORMAT_VERSION=1 +ENGINE_VERSION=1.7.0 +LOCK_DIR="${DATA_ROOT}/.kb-checkpoint-backup-lock" +META_DIR="" +checkpoint_dirs=() +checkpoint_cleanup_enabled=0 + +log() { + echo "INFO: $*" +} + +fail() { + echo "ERROR: $*" >&2 + return 1 +} + +read_property() { + local file=$1 + local key=$2 + awk -F= -v key="$key" ' + $0 !~ /^[[:space:]]*#/ { + lhs=$1 + gsub(/^[[:space:]]+|[[:space:]]+$/, "", lhs) + if (lhs == key) { + value=substr($0, index($0, "=") + 1) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + print value + exit + } + } + ' "$file" +} + +validate_direct_child() { + local path=$1 + local label=$2 + [[ "$path" == "${DATA_ROOT}/"* ]] || fail "${label} is outside ${DATA_ROOT}: ${path}" + [[ "$(dirname "$path")" == "$DATA_ROOT" ]] || fail "${label} is not a direct child of ${DATA_ROOT}: ${path}" + [[ "$(basename "$path")" =~ ^[A-Za-z0-9._-]+$ ]] || fail "${label} has an unsafe name: ${path}" +} + +validate_managed_path() { + local path=$1 + local label=$2 + local name + + validate_direct_child "$path" "$label" + name=$(basename "$path") + case "$name" in + graphs|docker|lost+found|snapshot_*|.kb-*) + fail "${label} uses reserved path ${path}" + ;; + esac +} + +cleanup() { + local path + + if (( checkpoint_cleanup_enabled == 1 )); then + while IFS= read -r -d '' path; do + checkpoint_dirs+=("$path") + done < <(find "$DATA_ROOT" -mindepth 1 -maxdepth 1 -type d -name 'snapshot_*' -print0) + fi + for path in "${checkpoint_dirs[@]:-}"; do + [[ -n "$path" ]] || continue + if [[ "$path" == "${DATA_ROOT}/snapshot_"* && "$(dirname "$path")" == "$DATA_ROOT" ]]; then + rm -rf -- "$path" + fi + done + [[ -n "$META_DIR" ]] && rm -rf -- "$META_DIR" + rmdir "$LOCK_DIR" 2>/dev/null || true +} + +on_exit() { + local code=$? + if (( code != 0 )); then + echo "checkpoint backup failed with exit code ${code}" >&2 + [[ -n "${DP_BACKUP_INFO_FILE:-}" ]] && touch "${DP_BACKUP_INFO_FILE}.exit" + fi + cleanup + exit "$code" +} +trap on_exit EXIT + +: "${DP_DATASAFED_BIN_PATH:?DP_DATASAFED_BIN_PATH is required}" +: "${DP_BACKUP_BASE_PATH:?DP_BACKUP_BASE_PATH is required}" +: "${DP_BACKUP_NAME:?DP_BACKUP_NAME is required}" +: "${DP_BACKUP_INFO_FILE:?DP_BACKUP_INFO_FILE is required}" +: "${DP_DB_HOST:?DP_DB_HOST is required}" +: "${DP_DB_USER:?DP_DB_USER is required}" +: "${DP_DB_PASSWORD:?DP_DB_PASSWORD is required}" + +export PATH="${PATH}:${DP_DATASAFED_BIN_PATH}" +export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" + +mkdir "$LOCK_DIR" 2>/dev/null || fail "another checkpoint backup is active on this PVC" +META_DIR=$(mktemp -d "${DATA_ROOT}/.kb-checkpoint-meta.XXXXXX") + +shopt -s nullglob +graph_configs=("${GRAPH_DIR}"/*.properties) +(( ${#graph_configs[@]} > 0 )) || fail "no persistent graph configuration found" + +preexisting=("${DATA_ROOT}"/snapshot_*) +(( ${#preexisting[@]} == 0 )) || fail "pre-existing snapshot_* directories must be resolved before backup" +checkpoint_cleanup_enabled=1 + +declare -a graph_names=() +declare -a config_hashes=() +declare -A path_owners=() +declare -A expected_checkpoints=() + +for graph_config in "${graph_configs[@]}"; do + graph_name=$(basename "$graph_config" .properties) + backend=$(read_property "$graph_config" backend) + store=$(read_property "$graph_config" store) + data_path=$(read_property "$graph_config" rocksdb.data_path) + wal_path=$(read_property "$graph_config" rocksdb.wal_path) + graph_factory=$(read_property "$graph_config" gremlin.graph) + + [[ "$graph_name" =~ ^[A-Za-z][A-Za-z0-9_]{0,47}$ ]] || fail "invalid graph name from config: ${graph_name}" + [[ "$backend" == "rocksdb" ]] || fail "graph ${graph_name} is not RocksDB" + [[ "${store,,}" == "${graph_name,,}" ]] || fail "graph ${graph_name} has mismatched store=${store:-}" + [[ "$graph_factory" == "org.apache.hugegraph.auth.HugeFactoryAuthProxy" ]] || fail "graph ${graph_name} is not using HugeFactoryAuthProxy" + [[ -n "$data_path" && -n "$wal_path" ]] || fail "graph ${graph_name} must declare RocksDB data and WAL paths" + validate_managed_path "$data_path" "graph ${graph_name} data path" + validate_managed_path "$wal_path" "graph ${graph_name} WAL path" + [[ "$data_path" != "$wal_path" ]] || fail "graph ${graph_name} data and WAL paths are identical" + + for owned_path in "$data_path" "$wal_path"; do + [[ -z "${path_owners[$owned_path]:-}" ]] || fail "path ${owned_path} is shared by ${path_owners[$owned_path]} and ${graph_name}" + path_owners[$owned_path]=$graph_name + done + + graph_names+=("$graph_name") + config_hashes+=("$(sha256sum "$graph_config" | awk '{print $1}')") + expected_checkpoints["snapshot_$(basename "$data_path")"]=$graph_name +done + +for graph_name in "${graph_names[@]}"; do + log "creating checkpoint for graph ${graph_name}" + curl --fail --silent --show-error \ + --user "${DP_DB_USER}:${DP_DB_PASSWORD}" \ + --request PUT \ + "http://${DP_DB_HOST}:${API_PORT}/graphspaces/DEFAULT/graphs/${graph_name}/snapshot_create" \ + >/dev/null +done + +while IFS= read -r -d '' checkpoint; do + checkpoint_dirs+=("$checkpoint") +done < <(find "$DATA_ROOT" -mindepth 1 -maxdepth 1 -type d -name 'snapshot_*' -print0 | sort -z) + +(( ${#checkpoint_dirs[@]} == ${#graph_names[@]} )) || fail "checkpoint directory count does not match graph count" + +declare -a checkpoint_names=() +for checkpoint in "${checkpoint_dirs[@]}"; do + checkpoint_name=$(basename "$checkpoint") + origin_name=${checkpoint_name#snapshot_} + [[ "$checkpoint_name" == snapshot_* && -n "$origin_name" ]] || fail "invalid checkpoint directory ${checkpoint_name}" + [[ "$checkpoint_name" =~ ^snapshot_[A-Za-z0-9._-]+$ ]] || fail "unsafe checkpoint directory ${checkpoint_name}" + [[ "$origin_name" =~ ^[A-Za-z0-9._-]+$ ]] || fail "unsafe checkpoint destination ${origin_name}" + [[ -n "${expected_checkpoints[$checkpoint_name]:-}" ]] || fail "checkpoint ${checkpoint_name} does not map to a graph data path" + [[ -z "$(find "$checkpoint" -type l -print -quit)" ]] || fail "checkpoint ${checkpoint_name} contains symbolic links" + [[ -z "$(find "$checkpoint" ! -type d ! -type f -print -quit)" ]] || fail "checkpoint ${checkpoint_name} contains a special file" + while IFS= read -r -d '' member; do + relative=${member#"${DATA_ROOT}/"} + [[ "$relative" =~ ^[A-Za-z0-9._/-]+$ ]] || fail "checkpoint ${checkpoint_name} contains an unsafe path: ${relative}" + done < <(find "$checkpoint" -mindepth 1 -print0) + [[ -n "$(find "$checkpoint" -type f -size +0c -print -quit)" ]] || fail "checkpoint ${checkpoint_name} contains no non-empty files" + checkpoint_names+=("$checkpoint_name") + unset 'expected_checkpoints[$checkpoint_name]' +done + +(( ${#expected_checkpoints[@]} == 0 )) || fail "one or more graph data paths have no checkpoint" + +manifest="${META_DIR}/manifest.properties" +checksums="${META_DIR}/checksums.sha256" +{ + echo "format.version=${FORMAT_VERSION}" + echo "engine=hugegraph" + echo "engine.version=${ENGINE_VERSION}" + echo "backup.type=full" + echo "graph.count=${#graph_names[@]}" + for i in "${!graph_names[@]}"; do + echo "graph.${i}.name=${graph_names[$i]}" + echo "graph.${i}.config=graphs/${graph_names[$i]}.properties" + echo "graph.${i}.config.sha256=${config_hashes[$i]}" + done + echo "checkpoint.count=${#checkpoint_names[@]}" + for i in "${!checkpoint_names[@]}"; do + echo "checkpoint.${i}.source=${checkpoint_names[$i]}" + echo "checkpoint.${i}.destination=${checkpoint_names[$i]#snapshot_}" + done +} >"$manifest" + +: >"$checksums" +for graph_config in "${graph_configs[@]}"; do + relative="graphs/$(basename "$graph_config")" + printf '%s %s\n' "$(sha256sum "$graph_config" | awk '{print $1}')" "$relative" >>"$checksums" +done +for checkpoint in "${checkpoint_dirs[@]}"; do + while IFS= read -r file; do + relative=${file#"${DATA_ROOT}/"} + printf '%s %s\n' "$(sha256sum "$file" | awk '{print $1}')" "$relative" >>"$checksums" + done < <(find "$checkpoint" -type f -print | LC_ALL=C sort) +done + +cp "$manifest" "${META_DIR}/manifest.upload" +cp "$checksums" "${META_DIR}/checksums.upload" +datasafed push "${META_DIR}/manifest.upload" manifest.properties +datasafed push "${META_DIR}/checksums.upload" checksums.sha256 + +log "uploading checkpoint payload" +graph_entries=() +for graph_config in "${graph_configs[@]}"; do + graph_entries+=("graphs/$(basename "$graph_config")") +done +tar -C "$DATA_ROOT" -czf - "${graph_entries[@]}" "${checkpoint_names[@]}" \ + -C "$META_DIR" manifest.properties checksums.sha256 \ + | datasafed push - payload.tar.gz + +total_size=$(datasafed stat / | awk '/TotalSize/ {print $2; exit}') +[[ -n "$total_size" ]] || fail "datasafed did not report backup size" +printf '{"totalSize":"%s"}\n' "$total_size" >"$DP_BACKUP_INFO_FILE" +log "checkpoint backup completed for ${#graph_names[@]} graph(s)" diff --git a/addons/hugegraph/scripts/restore.sh b/addons/hugegraph/scripts/restore.sh new file mode 100644 index 0000000000..14a49a850b --- /dev/null +++ b/addons/hugegraph/scripts/restore.sh @@ -0,0 +1,321 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +DATA_ROOT=${DATA_ROOT:-/hugegraph-data} +FORMAT_VERSION=1 +ENGINE_VERSION=1.7.0 +WORK_DIR="" +STAGING_DIR="${DATA_ROOT}/.kb-restore-staging" +IN_PROGRESS_MARKER="${DATA_ROOT}/.kb-restore-in-progress" +COMPLETE_MARKER="${DATA_ROOT}/.kb-restored-backup" +INIT_MARKER="${DATA_ROOT}/docker/init_complete" + +log() { + echo "INFO: $*" +} + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +manifest_get() { + local key=$1 + awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "${WORK_DIR}/manifest.properties" +} + +read_property() { + local file=$1 + local key=$2 + awk -F= -v key="$key" ' + $0 !~ /^[[:space:]]*#/ { + lhs=$1 + gsub(/^[[:space:]]+|[[:space:]]+$/, "", lhs) + if (lhs == key) { + value=substr($0, index($0, "=") + 1) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + print value + exit + } + } + ' "$file" +} + +validate_count() { + local value=$1 + local label=$2 + [[ "$value" =~ ^[1-9][0-9]*$ ]] || fail "${label} must be a positive integer, got ${value:-}" +} + +validate_managed_path() { + local path=$1 + local label=$2 + local name + + [[ "$path" == "${DATA_ROOT}/"* ]] || fail "${label} is outside ${DATA_ROOT}: ${path}" + [[ "$(dirname "$path")" == "$DATA_ROOT" ]] || fail "${label} is not a direct child of ${DATA_ROOT}: ${path}" + name=$(basename "$path") + [[ "$name" =~ ^[A-Za-z0-9._-]+$ ]] || fail "${label} has an unsafe name: ${path}" + case "$name" in + graphs|docker|lost+found|snapshot_*|.kb-*) + fail "${label} uses reserved path ${path}" + ;; + esac +} + +validate_member() { + local member=$1 + local root + [[ -n "$member" ]] || fail "payload contains an empty member name" + [[ "$member" != /* ]] || fail "payload contains an absolute path: ${member}" + [[ "$member" != ".." && "$member" != ../* && "$member" != */../* && "$member" != */.. ]] || fail "payload contains path traversal: ${member}" + [[ "$member" =~ ^[A-Za-z0-9._/-]+$ ]] || fail "payload contains an unsafe member name: ${member}" + case "$member" in + graphs|graphs/|manifest.properties|checksums.sha256) + ;; + graphs/*.properties) + [[ -n "${graph_config_members[$member]:-}" ]] || fail "payload contains an unlisted graph config: ${member}" + ;; + *) + root=${member%%/*} + [[ -n "${checkpoint_source_members[$root]:-}" ]] || fail "payload contains an unlisted checkpoint member: ${member}" + ;; + esac +} + +cleanup() { + [[ -n "$WORK_DIR" ]] && rm -rf -- "$WORK_DIR" + rm -rf -- "$STAGING_DIR" +} +trap cleanup EXIT + +: "${DP_DATASAFED_BIN_PATH:?DP_DATASAFED_BIN_PATH is required}" +: "${DP_BACKUP_BASE_PATH:?DP_BACKUP_BASE_PATH is required}" +: "${DP_BACKUP_NAME:?DP_BACKUP_NAME is required}" + +export PATH="${PATH}:${DP_DATASAFED_BIN_PATH}" +export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" + +WORK_DIR=$(mktemp -d /tmp/hugegraph-restore.XXXXXX) +datasafed pull manifest.properties "${WORK_DIR}/manifest.properties" +datasafed pull checksums.sha256 "${WORK_DIR}/checksums.sha256" +datasafed pull payload.tar.gz "${WORK_DIR}/payload.tar.gz" + +[[ "$(manifest_get format.version)" == "$FORMAT_VERSION" ]] || fail "unsupported checkpoint format" +[[ "$(manifest_get engine)" == "hugegraph" ]] || fail "artifact is not a HugeGraph backup" +[[ "$(manifest_get engine.version)" == "$ENGINE_VERSION" ]] || fail "HugeGraph version mismatch" +[[ "$(manifest_get backup.type)" == "full" ]] || fail "only full checkpoint backup is supported" + +graph_count=$(manifest_get graph.count) +checkpoint_count=$(manifest_get checkpoint.count) +validate_count "$graph_count" graph.count +validate_count "$checkpoint_count" checkpoint.count +(( checkpoint_count == graph_count )) || fail "checkpoint count does not match graph count" + +declare -a checkpoint_sources=() +declare -a checkpoint_destinations=() +declare -A destinations=() +declare -A checkpoint_source_members=() +declare -A graph_names=() +declare -A graph_config_members=() +declare -A archive_members=() +declare -A checksum_members=() + +for ((i = 0; i < checkpoint_count; i++)); do + source_name=$(manifest_get "checkpoint.${i}.source") + destination=$(manifest_get "checkpoint.${i}.destination") + [[ "$source_name" =~ ^snapshot_[A-Za-z0-9._-]+$ ]] || fail "unsafe checkpoint source ${source_name:-}" + [[ "$destination" =~ ^[A-Za-z0-9._-]+$ ]] || fail "unsafe checkpoint destination ${destination:-}" + [[ "$destination" == "${source_name#snapshot_}" ]] || fail "checkpoint destination does not match source ${source_name}" + [[ -z "${destinations[$destination]:-}" ]] || fail "duplicate checkpoint destination ${destination}" + destinations[$destination]=1 + checkpoint_source_members[$source_name]=1 + checkpoint_sources+=("$source_name") + checkpoint_destinations+=("$destination") +done + +for ((i = 0; i < graph_count; i++)); do + graph_name=$(manifest_get "graph.${i}.name") + graph_config=$(manifest_get "graph.${i}.config") + expected_hash=$(manifest_get "graph.${i}.config.sha256") + [[ "$graph_name" =~ ^[A-Za-z][A-Za-z0-9_]{0,47}$ ]] || fail "unsafe graph name ${graph_name:-}" + [[ -z "${graph_names[$graph_name]:-}" ]] || fail "duplicate graph name ${graph_name}" + graph_names[$graph_name]=1 + [[ "$graph_config" == "graphs/${graph_name}.properties" ]] || fail "graph config path mismatch for ${graph_name}" + graph_config_members[$graph_config]=1 + [[ "$expected_hash" =~ ^[a-f0-9]{64}$ ]] || fail "invalid graph config hash for ${graph_name}" +done + +log "validating checkpoint payload members" +tar -tzf "${WORK_DIR}/payload.tar.gz" >"${WORK_DIR}/payload.list" +tar -tvzf "${WORK_DIR}/payload.tar.gz" >"${WORK_DIR}/payload.verbose" +while IFS= read -r line; do + type=${line:0:1} + [[ "$type" == "-" || "$type" == "d" ]] || fail "payload contains a non-regular member" +done <"${WORK_DIR}/payload.verbose" +while IFS= read -r member; do + validate_member "$member" + [[ -z "${archive_members[$member]:-}" ]] || fail "payload contains a duplicate member: ${member}" + archive_members[$member]=1 +done <"${WORK_DIR}/payload.list" +while IFS= read -r checksum_line; do + [[ "$checksum_line" =~ ^([a-f0-9]{64})[[:space:]][[:space:]]([A-Za-z0-9._/-]+)$ ]] || fail "checksum list contains an invalid entry" + relative=${BASH_REMATCH[2]} + validate_member "$relative" + if [[ "$relative" == "manifest.properties" || "$relative" == "checksums.sha256" || + "$relative" == "graphs" || "$relative" == "graphs/" || + "$relative" =~ ^snapshot_[A-Za-z0-9._-]+/?$ ]]; then + fail "checksum list references a non-payload file: ${relative}" + fi + [[ -n "${archive_members[$relative]:-}" ]] || fail "checksum list references a missing archive member: ${relative}" + [[ -z "${checksum_members[$relative]:-}" ]] || fail "checksum list contains a duplicate member: ${relative}" + checksum_members[$relative]=1 +done <"${WORK_DIR}/checksums.sha256" + +rm -rf -- "$STAGING_DIR" +mkdir -p "$STAGING_DIR" +tar -xzf "${WORK_DIR}/payload.tar.gz" -C "$STAGING_DIR" --no-same-owner --no-same-permissions + +cmp -s "${WORK_DIR}/manifest.properties" "${STAGING_DIR}/manifest.properties" || fail "inner and outer manifests differ" +cmp -s "${WORK_DIR}/checksums.sha256" "${STAGING_DIR}/checksums.sha256" || fail "inner and outer checksum lists differ" + +while IFS= read -r -d '' payload_file; do + relative=${payload_file#"${STAGING_DIR}/"} + case "$relative" in + manifest.properties|checksums.sha256) + continue + ;; + esac + [[ -n "${checksum_members[$relative]:-}" ]] || fail "payload file is not checksummed: ${relative}" +done < <(find "$STAGING_DIR" -type f -print0) + +( + cd "$STAGING_DIR" + sha256sum -c checksums.sha256 +) + +declare -A graph_data_destinations=() +declare -A managed_paths=() +declare -a wal_destinations=() + +for ((i = 0; i < graph_count; i++)); do + graph_name=$(manifest_get "graph.${i}.name") + graph_config=$(manifest_get "graph.${i}.config") + expected_hash=$(manifest_get "graph.${i}.config.sha256") + [[ "$(sha256sum "${STAGING_DIR}/${graph_config}" | awk '{print $1}')" == "$expected_hash" ]] || fail "graph config hash mismatch for ${graph_name}" + + backend=$(read_property "${STAGING_DIR}/${graph_config}" backend) + store=$(read_property "${STAGING_DIR}/${graph_config}" store) + graph_factory=$(read_property "${STAGING_DIR}/${graph_config}" gremlin.graph) + data_path=$(read_property "${STAGING_DIR}/${graph_config}" rocksdb.data_path) + wal_path=$(read_property "${STAGING_DIR}/${graph_config}" rocksdb.wal_path) + [[ "$backend" == "rocksdb" ]] || fail "graph ${graph_name} is not RocksDB" + [[ "${store,,}" == "${graph_name,,}" ]] || fail "graph ${graph_name} has mismatched store=${store:-}" + [[ "$graph_factory" == "org.apache.hugegraph.auth.HugeFactoryAuthProxy" ]] || fail "graph ${graph_name} is not using HugeFactoryAuthProxy" + [[ -n "$data_path" && -n "$wal_path" ]] || fail "graph ${graph_name} must declare RocksDB data and WAL paths" + validate_managed_path "$data_path" "graph ${graph_name} data path" + validate_managed_path "$wal_path" "graph ${graph_name} WAL path" + [[ "$data_path" != "$wal_path" ]] || fail "graph ${graph_name} data and WAL paths are identical" + for managed_path in "$data_path" "$wal_path"; do + [[ -z "${managed_paths[$managed_path]:-}" ]] || fail "path ${managed_path} is shared by ${managed_paths[$managed_path]} and ${graph_name}" + managed_paths[$managed_path]=$graph_name + done + data_destination=$(basename "$data_path") + [[ -n "${destinations[$data_destination]:-}" ]] || fail "graph ${graph_name} data path has no checkpoint" + graph_data_destinations[$data_destination]=1 + wal_destinations+=("$(basename "$wal_path")") +done + +(( ${#graph_data_destinations[@]} == ${#destinations[@]} )) || fail "manifest contains a checkpoint not owned by a graph" + +[[ -z "$(find "$STAGING_DIR" ! -type d ! -type f -print -quit)" ]] || fail "payload extracted a special file" + +for source_name in "${checkpoint_sources[@]}"; do + [[ -d "${STAGING_DIR}/${source_name}" ]] || fail "checkpoint directory ${source_name} is missing" + [[ -n "$(find "${STAGING_DIR}/${source_name}" -type f -size +0c -print -quit)" ]] || fail "checkpoint ${source_name} is empty" +done + +validate_restored_layout() { + local i graph_name graph_config expected_hash destination checksum relative + local source_root remainder actual + + [[ -d "${DATA_ROOT}/graphs" ]] || fail "restore marker exists but graph configs are missing" + [[ -f "$INIT_MARKER" ]] || fail "restore marker exists but HugeGraph init marker is missing" + for ((i = 0; i < graph_count; i++)); do + graph_name=$(manifest_get "graph.${i}.name") + graph_config=$(manifest_get "graph.${i}.config") + expected_hash=$(manifest_get "graph.${i}.config.sha256") + [[ -f "${DATA_ROOT}/${graph_config}" ]] || fail "restored graph config ${graph_config} is missing" + [[ "$(sha256sum "${DATA_ROOT}/${graph_config}" | awk '{print $1}')" == "$expected_hash" ]] || fail "restored graph config hash mismatch for ${graph_name}" + done + for destination in "${checkpoint_destinations[@]}"; do + [[ -d "${DATA_ROOT}/${destination}" ]] || fail "restored checkpoint destination ${destination} is missing" + [[ -n "$(find "${DATA_ROOT}/${destination}" -type f -size +0c -print -quit)" ]] || fail "restored checkpoint destination ${destination} is empty" + done + while read -r checksum relative; do + [[ "$checksum" =~ ^[a-f0-9]{64}$ && -n "$relative" ]] || fail "invalid checksum entry in completed restore" + if [[ "$relative" == snapshot_*/* ]]; then + source_root=${relative%%/*} + remainder=${relative#*/} + [[ -n "${checkpoint_source_members[$source_root]:-}" ]] || fail "checksum references an unknown checkpoint ${source_root}" + actual="${DATA_ROOT}/${source_root#snapshot_}/${remainder}" + else + actual="${DATA_ROOT}/${relative}" + fi + [[ -f "$actual" ]] || fail "restored file ${actual} is missing" + [[ "$(sha256sum "$actual" | awk '{print $1}')" == "$checksum" ]] || fail "restored file checksum mismatch: ${actual}" + done <"${WORK_DIR}/checksums.sha256" +} + +if [[ -f "$COMPLETE_MARKER" ]] && [[ "$(<"$COMPLETE_MARKER")" == "$DP_BACKUP_NAME" ]]; then + validate_restored_layout + if [[ -f "$IN_PROGRESS_MARKER" ]]; then + [[ "$(<"$IN_PROGRESS_MARKER")" == "$DP_BACKUP_NAME" ]] || fail "completed restore has a conflicting in-progress marker" + rm -f -- "$IN_PROGRESS_MARKER" + fi + log "backup ${DP_BACKUP_NAME} is already restored" + exit 0 +fi + +if [[ -f "$IN_PROGRESS_MARKER" ]]; then + [[ "$(<"$IN_PROGRESS_MARKER")" == "$DP_BACKUP_NAME" ]] || fail "PVC contains an unfinished restore from another backup" +else + [[ ! -e "$COMPLETE_MARKER" ]] || fail "PVC was restored from another backup" + while IFS= read -r -d '' existing; do + case "$(basename "$existing")" in + lost+found|.kb-restore-staging) + ;; + *) + fail "target PVC is not empty: ${existing}" + ;; + esac + done < <(find "$DATA_ROOT" -mindepth 1 -maxdepth 1 -print0) +fi + +printf '%s\n' "$DP_BACKUP_NAME" >"$IN_PROGRESS_MARKER" + +rm -rf -- "${DATA_ROOT}/graphs" +for destination in "${checkpoint_destinations[@]}"; do + rm -rf -- "${DATA_ROOT:?}/${destination:?}" +done +for destination in "${wal_destinations[@]}"; do + rm -rf -- "${DATA_ROOT:?}/${destination:?}" +done +rm -rf -- "${DATA_ROOT}/docker" + +for i in "${!checkpoint_sources[@]}"; do + mv "${STAGING_DIR}/${checkpoint_sources[$i]}" "${DATA_ROOT}/${checkpoint_destinations[$i]}" +done +mv "${STAGING_DIR}/graphs" "${DATA_ROOT}/graphs" +mkdir -p "$(dirname "$INIT_MARKER")" +touch "$INIT_MARKER" + +sync + +printf '%s\n' "$DP_BACKUP_NAME" >"${COMPLETE_MARKER}.tmp" +mv "${COMPLETE_MARKER}.tmp" "$COMPLETE_MARKER" +sync +rm -f -- "$IN_PROGRESS_MARKER" +sync +log "restored ${graph_count} graph(s) from checkpoint backup ${DP_BACKUP_NAME}" diff --git a/addons/hugegraph/scripts/start.sh b/addons/hugegraph/scripts/start.sh new file mode 100644 index 0000000000..f2ce089ea8 --- /dev/null +++ b/addons/hugegraph/scripts/start.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +SERVER_HOME=${HUGEGRAPH_SERVER_HOME:-/hugegraph-server} +DATA_ROOT=${HUGEGRAPH_DATA_ROOT:-/hugegraph-data} +GRAPH_DIR="${DATA_ROOT}/graphs" +REST_CONFIG="${SERVER_HOME}/conf/rest-server.properties" +DEFAULT_GRAPH_CONFIG="${SERVER_HOME}/conf/graphs/hugegraph.properties" + +log() { + echo "INFO: $*" +} + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +read_property() { + local file=$1 + local key=$2 + awk -F= -v key="$key" ' + $0 !~ /^[[:space:]]*#/ { + lhs=$1 + gsub(/^[[:space:]]+|[[:space:]]+$/, "", lhs) + if (lhs == key) { + value=substr($0, index($0, "=") + 1) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + print value + exit + } + } + ' "$file" +} + +set_property() { + local file=$1 + local key=$2 + local value=$3 + local escaped_key=${key//./\.} + + if grep -Eq "^[[:space:]]*${escaped_key}[[:space:]]*=" "$file"; then + sed -i -E "s#^[[:space:]]*${escaped_key}[[:space:]]*=.*#${key}=${value}#" "$file" + else + printf '%s=%s\n' "$key" "$value" >>"$file" + fi +} + +validate_direct_child() { + local path=$1 + local label=$2 + [[ "$path" == "${DATA_ROOT}/"* ]] || fail "${label} must be under ${DATA_ROOT}: ${path}" + [[ "$(dirname "$path")" == "$DATA_ROOT" ]] || fail "${label} must be a direct child of ${DATA_ROOT}: ${path}" + [[ "$(basename "$path")" =~ ^[A-Za-z0-9._-]+$ ]] || fail "${label} has an unsafe directory name: ${path}" +} + +validate_managed_path() { + local path=$1 + local label=$2 + local name + + validate_direct_child "$path" "$label" + name=$(basename "$path") + case "$name" in + graphs|docker|lost+found|snapshot_*|.kb-*) + fail "${label} uses reserved path ${path}" + ;; + esac +} + +mkdir -p "$GRAPH_DIR" "${DATA_ROOT}/docker" + +if ! compgen -G "${GRAPH_DIR}/*.properties" >/dev/null; then + log "installing the default graph configuration into persistent storage" + cp "$DEFAULT_GRAPH_CONFIG" "${GRAPH_DIR}/hugegraph.properties" +fi + +set_property "$REST_CONFIG" restserver.url http://0.0.0.0:8080 +set_property "$REST_CONFIG" graphs "$GRAPH_DIR" + +shopt -s nullglob +graph_configs=("${GRAPH_DIR}"/*.properties) +(( ${#graph_configs[@]} > 0 )) || fail "no graph configuration found in ${GRAPH_DIR}" +declare -A path_owners=() + +for graph_config in "${graph_configs[@]}"; do + graph_name=$(basename "$graph_config" .properties) + backend=$(read_property "$graph_config" backend) + store=$(read_property "$graph_config" store) + data_path=$(read_property "$graph_config" rocksdb.data_path) + wal_path=$(read_property "$graph_config" rocksdb.wal_path) + + [[ "$graph_name" =~ ^[A-Za-z][A-Za-z0-9_]{0,47}$ ]] || fail "invalid graph name from config: ${graph_name}" + [[ "$backend" == "rocksdb" ]] || fail "graph ${graph_name} uses unsupported backend ${backend:-}" + [[ "${store,,}" == "${graph_name,,}" ]] || fail "graph config ${graph_name} must use store=${graph_name}, got ${store:-}" + + if [[ "$graph_name" == "hugegraph" ]]; then + data_path=${data_path:-${DATA_ROOT}/rocksdb} + wal_path=${wal_path:-${DATA_ROOT}/wal} + set_property "$graph_config" rocksdb.data_path "$data_path" + set_property "$graph_config" rocksdb.wal_path "$wal_path" + else + [[ -n "$data_path" && -n "$wal_path" ]] || fail "graph ${graph_name} must set persistent rocksdb.data_path and rocksdb.wal_path" + fi + + validate_managed_path "$data_path" "graph ${graph_name} data path" + validate_managed_path "$wal_path" "graph ${graph_name} WAL path" + [[ "$data_path" != "$wal_path" ]] || fail "graph ${graph_name} data and WAL paths must differ" + for managed_path in "$data_path" "$wal_path"; do + [[ -z "${path_owners[$managed_path]:-}" ]] || fail "path ${managed_path} is shared by ${path_owners[$managed_path]} and ${graph_name}" + path_owners[$managed_path]=$graph_name + done + + set_property "$graph_config" gremlin.graph org.apache.hugegraph.auth.HugeFactoryAuthProxy +done + +docker_marker="${SERVER_HOME}/docker" +if [[ -e "$docker_marker" && ! -L "$docker_marker" ]]; then + if [[ -d "$docker_marker" && -z "$(find "$docker_marker" -mindepth 1 -maxdepth 1 -print -quit)" ]]; then + rmdir "$docker_marker" + else + fail "${docker_marker} exists and is not an empty directory or symlink" + fi +fi +ln -sfn "${DATA_ROOT}/docker" "$docker_marker" + +log "starting HugeGraph with ${#graph_configs[@]} persistent graph configuration(s)" +cd "$SERVER_HOME" +exec /usr/bin/dumb-init -- ./docker-entrypoint.sh diff --git a/addons/hugegraph/templates/_helpers.tpl b/addons/hugegraph/templates/_helpers.tpl new file mode 100644 index 0000000000..988cf1d32b --- /dev/null +++ b/addons/hugegraph/templates/_helpers.tpl @@ -0,0 +1,53 @@ +{{/* Chart and resource names. */}} +{{- define "hugegraph.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "hugegraph.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "hugegraph.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hugegraph.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "hugegraph.labels" -}} +helm.sh/chart: {{ include "hugegraph.chart" . }} +{{ include "hugegraph.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "hugegraph.apiVersion" -}} +kubeblocks.io/crd-api-version: apps.kubeblocks.io/v1 +{{- end }} + +{{- define "hugegraph.annotations" -}} +{{ include "kblib.helm.resourcePolicy" . }} +{{ include "hugegraph.apiVersion" . }} +{{- end }} + +{{- define "hugegraph.cmpdName" -}} +hugegraph-{{ .Chart.Version }} +{{- end }} + +{{- define "hugegraph.cmpdPattern" -}} +^hugegraph- +{{- end }} + +{{- define "hugegraph.scriptsTemplateName" -}} +hugegraph-scripts-template +{{- end }} + +{{- define "hugegraph.actionSetName" -}} +hugegraph-checkpoint-br +{{- end }} + +{{- define "hugegraph.backupPolicyTemplateName" -}} +hugegraph-backup-policy-template +{{- end }} + +{{- define "hugegraph.image" -}} +{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} +{{- end }} diff --git a/addons/hugegraph/templates/actionset.yaml b/addons/hugegraph/templates/actionset.yaml new file mode 100644 index 0000000000..82f738ba58 --- /dev/null +++ b/addons/hugegraph/templates/actionset.yaml @@ -0,0 +1,34 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: ActionSet +metadata: + name: {{ include "hugegraph.actionSetName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} +spec: + backupType: Full + env: + - name: DATA_ROOT + value: {{ .Values.dataMountPath }} + - name: API_PORT + value: "8080" + backup: + backupData: + image: {{ include "hugegraph.image" . }} + runOnTargetPodNode: true + command: + - /bin/bash + - -c + - | + {{- .Files.Get "scripts/backup.sh" | nindent 10 }} + syncProgress: + enabled: true + intervalSeconds: 10 + restore: + prepareData: + image: {{ include "hugegraph.image" . }} + runOnTargetPodNode: true + command: + - /bin/bash + - -c + - | + {{- .Files.Get "scripts/restore.sh" | nindent 10 }} diff --git a/addons/hugegraph/templates/backuppolicytemplate.yaml b/addons/hugegraph/templates/backuppolicytemplate.yaml new file mode 100644 index 0000000000..1bf54714f7 --- /dev/null +++ b/addons/hugegraph/templates/backuppolicytemplate.yaml @@ -0,0 +1,29 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: BackupPolicyTemplate +metadata: + name: {{ include "hugegraph.backupPolicyTemplateName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} +spec: + serviceKind: hugegraph + compDefs: + - {{ include "hugegraph.cmpdPattern" . }} + target: + role: "" + strategy: All + account: admin + backupMethods: + - name: checkpoint + snapshotVolumes: false + actionSetName: {{ include "hugegraph.actionSetName" . }} + targetVolumes: + volumes: + - data + volumeMounts: + - name: data + mountPath: {{ .Values.dataMountPath }} + schedules: + - backupMethod: checkpoint + enabled: false + cronExpression: "0 18 * * 0" + retentionPeriod: 7d diff --git a/addons/hugegraph/templates/clusterdefinition.yaml b/addons/hugegraph/templates/clusterdefinition.yaml new file mode 100644 index 0000000000..9e7bd5a2dd --- /dev/null +++ b/addons/hugegraph/templates/clusterdefinition.yaml @@ -0,0 +1,15 @@ +apiVersion: apps.kubeblocks.io/v1 +kind: ClusterDefinition +metadata: + name: hugegraph + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + annotations: + {{- include "hugegraph.apiVersion" . | nindent 4 }} +spec: + topologies: + - name: standalone + default: true + components: + - name: server + compDef: {{ include "hugegraph.cmpdPattern" . }} diff --git a/addons/hugegraph/templates/cmpd.yaml b/addons/hugegraph/templates/cmpd.yaml new file mode 100644 index 0000000000..9d258d87dd --- /dev/null +++ b/addons/hugegraph/templates/cmpd.yaml @@ -0,0 +1,120 @@ +apiVersion: apps.kubeblocks.io/v1 +kind: ComponentDefinition +metadata: + name: {{ include "hugegraph.cmpdName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + annotations: + apps.kubeblocks.io/skip-immutable-check: "true" + {{- include "hugegraph.annotations" . | nindent 4 }} +spec: + provider: kubeblocks + description: Apache HugeGraph 1.7.0 standalone server with RocksDB. + serviceKind: hugegraph + serviceVersion: {{ .Values.serviceVersion }} + minReadySeconds: 10 + replicasLimit: + minReplicas: 1 + maxReplicas: 1 + services: + - name: default + spec: + ports: + - name: http + port: 8080 + targetPort: http + - name: gremlin + port: 8182 + targetPort: gremlin + scripts: + - name: hugegraph-scripts + template: {{ include "hugegraph.scriptsTemplateName" . }} + namespace: {{ .Release.Namespace }} + volumeName: scripts + defaultMode: 0555 + volumes: + - name: data + systemAccounts: + - name: admin + initAccount: true + passwordGenerationPolicy: + length: 16 + numDigits: 4 + numSymbols: 0 + letterCase: MixedCases + vars: + - name: ADMIN_USER + valueFrom: + credentialVarRef: + compDef: {{ include "hugegraph.cmpdName" . }} + name: admin + optional: false + username: Required + - name: ADMIN_PASSWORD + valueFrom: + credentialVarRef: + compDef: {{ include "hugegraph.cmpdName" . }} + name: admin + optional: false + password: Required + logConfigs: + - name: running + filePathPattern: {{ .Values.logConfigs.running }} + runtime: + containers: + - name: hugegraph + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + runAsUser: 0 + runAsGroup: 0 + command: + - /bin/bash + - /scripts/start.sh + env: + - name: PASSWORD + value: $(ADMIN_PASSWORD) + - name: HUGEGRAPH_DATA_ROOT + value: {{ .Values.dataMountPath }} + ports: + - name: http + containerPort: 8080 + - name: gremlin + containerPort: 8182 + startupProbe: + exec: + command: + - /bin/bash + - -ec + - >- + curl --fail --silent --show-error --user "${ADMIN_USER}:${ADMIN_PASSWORD}" + http://127.0.0.1:8080/graphspaces/DEFAULT/graphs >/dev/null + failureThreshold: 60 + periodSeconds: 5 + timeoutSeconds: 3 + readinessProbe: + exec: + command: + - /bin/bash + - -ec + - >- + curl --fail --silent --show-error --user "${ADMIN_USER}:${ADMIN_PASSWORD}" + http://127.0.0.1:8080/graphspaces/DEFAULT/graphs >/dev/null + failureThreshold: 3 + periodSeconds: 10 + timeoutSeconds: 3 + livenessProbe: + exec: + command: + - /bin/bash + - -ec + - >- + curl --fail --silent --show-error --user "${ADMIN_USER}:${ADMIN_PASSWORD}" + http://127.0.0.1:8080/graphspaces/DEFAULT/graphs >/dev/null + failureThreshold: 6 + periodSeconds: 15 + timeoutSeconds: 3 + volumeMounts: + - name: data + mountPath: {{ .Values.dataMountPath }} + - name: scripts + mountPath: /scripts diff --git a/addons/hugegraph/templates/cmpv.yaml b/addons/hugegraph/templates/cmpv.yaml new file mode 100644 index 0000000000..276e22fd07 --- /dev/null +++ b/addons/hugegraph/templates/cmpv.yaml @@ -0,0 +1,19 @@ +apiVersion: apps.kubeblocks.io/v1 +kind: ComponentVersion +metadata: + name: hugegraph + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + annotations: + {{- include "hugegraph.apiVersion" . | nindent 4 }} +spec: + compatibilityRules: + - compDefs: + - {{ include "hugegraph.cmpdPattern" . }} + releases: + - hugegraph-1.7.0 + releases: + - name: hugegraph-1.7.0 + serviceVersion: 1.7.0 + images: + hugegraph: {{ include "hugegraph.image" . }} diff --git a/addons/hugegraph/templates/script-template.yaml b/addons/hugegraph/templates/script-template.yaml new file mode 100644 index 0000000000..19a7826599 --- /dev/null +++ b/addons/hugegraph/templates/script-template.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hugegraph.scriptsTemplateName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} +data: + start.sh: |- + {{- .Files.Get "scripts/start.sh" | nindent 4 }} diff --git a/addons/hugegraph/tests/contract_test.sh b/addons/hugegraph/tests/contract_test.sh new file mode 100644 index 0000000000..4f816ec16b --- /dev/null +++ b/addons/hugegraph/tests/contract_test.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +ADDON_DIR="${ROOT_DIR}/addons/hugegraph" +CLUSTER_DIR="${ROOT_DIR}/addons-cluster/hugegraph" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_file() { + [[ -f "$1" ]] || fail "missing file: $1" +} + +assert_contains() { + local file=$1 + local pattern=$2 + rg -q -- "$pattern" "$file" || fail "$file does not contain: $pattern" +} + +assert_not_contains_tree() { + local pattern=$1 + if rg -n -i -- "$pattern" \ + "${ADDON_DIR}/templates" \ + "${ADDON_DIR}/values.yaml" \ + "${ROOT_DIR}/examples/hugegraph"; then + fail "forbidden pattern found: $pattern" + fi +} + +required_files=( + "${ADDON_DIR}/Chart.yaml" + "${ADDON_DIR}/values.yaml" + "${ADDON_DIR}/templates/cmpd.yaml" + "${ADDON_DIR}/templates/cmpv.yaml" + "${ADDON_DIR}/templates/clusterdefinition.yaml" + "${ADDON_DIR}/templates/backuppolicytemplate.yaml" + "${ADDON_DIR}/templates/actionset.yaml" + "${ADDON_DIR}/templates/script-template.yaml" + "${ADDON_DIR}/scripts/start.sh" + "${ADDON_DIR}/scripts/backup.sh" + "${ADDON_DIR}/scripts/restore.sh" + "${ADDON_DIR}/tests/scripts_test.sh" + "${CLUSTER_DIR}/Chart.yaml" + "${CLUSTER_DIR}/templates/cluster.yaml" + "${ROOT_DIR}/examples/hugegraph/cluster.yaml" + "${ROOT_DIR}/examples/hugegraph/backup.yaml" + "${ROOT_DIR}/examples/hugegraph/restore.yaml" +) + +for file in "${required_files[@]}"; do + assert_file "$file" +done + +bash -n \ + "${ADDON_DIR}/scripts/start.sh" \ + "${ADDON_DIR}/scripts/backup.sh" \ + "${ADDON_DIR}/scripts/restore.sh" + +assert_contains "${ADDON_DIR}/scripts/backup.sh" 'snapshot_create' +assert_contains "${ADDON_DIR}/scripts/backup.sh" 'manifest.properties' +assert_contains "${ADDON_DIR}/scripts/backup.sh" 'checksums.sha256' +assert_contains "${ADDON_DIR}/scripts/backup.sh" 'DP_BACKUP_INFO_FILE' +assert_contains "${ADDON_DIR}/scripts/restore.sh" 'sha256sum -c' +assert_contains "${ADDON_DIR}/scripts/restore.sh" 'restore-in-progress' +assert_contains "${ADDON_DIR}/scripts/restore.sh" 'payload.tar.gz' +assert_contains "${ADDON_DIR}/templates/backuppolicytemplate.yaml" 'snapshotVolumes: false' +assert_contains "${ADDON_DIR}/templates/backuppolicytemplate.yaml" 'name: checkpoint' +assert_contains "${ADDON_DIR}/templates/backuppolicytemplate.yaml" 'account: admin' +assert_contains "${ADDON_DIR}/templates/actionset.yaml" 'prepareData:' +assert_contains "${ADDON_DIR}/templates/actionset.yaml" 'runOnTargetPodNode: true' + +assert_not_contains_tree 'volume[-_ ]snapshot' +assert_not_contains_tree 'snapshotVolumes:[[:space:]]*true' + +helm dependency build "${ADDON_DIR}" >/dev/null +helm dependency build "${CLUSTER_DIR}" >/dev/null +helm lint "${ADDON_DIR}" +helm lint "${CLUSTER_DIR}" + +definition_render=$(mktemp) +cluster_render=$(mktemp) +trap 'rm -f "${definition_render}" "${cluster_render}"' EXIT + +helm template hugegraph-def "${ADDON_DIR}" --namespace kb-system >"${definition_render}" +helm template hugegraph "${CLUSTER_DIR}" --namespace demo >"${cluster_render}" + +for kind in ClusterDefinition ComponentDefinition ComponentVersion BackupPolicyTemplate ActionSet; do + rg -q "^kind: ${kind}$" "${definition_render}" || fail "render misses ${kind}" +done + +assert_contains "${definition_render}" 'serviceVersion: [\"]?1\.7\.0[\"]?' +assert_contains "${definition_render}" 'image: docker.io/hugegraph/hugegraph:1.7.0' +assert_contains "${definition_render}" 'mountPath: /hugegraph-data' +assert_contains "${definition_render}" 'containerPort: 8080' +assert_contains "${definition_render}" 'containerPort: 8182' +assert_contains "${cluster_render}" 'clusterDef: hugegraph' +assert_contains "${cluster_render}" 'topology: standalone' +assert_contains "${cluster_render}" 'replicas: 1' +assert_contains "${cluster_render}" 'serviceVersion: [\"]?1\.7\.0[\"]?' + +"${ADDON_DIR}/tests/scripts_test.sh" + +echo "HugeGraph addon offline contracts passed" diff --git a/addons/hugegraph/tests/scripts_test.sh b/addons/hugegraph/tests/scripts_test.sh new file mode 100755 index 0000000000..be0d33f55b --- /dev/null +++ b/addons/hugegraph/tests/scripts_test.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TEST_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ADDON_DIR=$(cd "${TEST_DIR}/.." && pwd) +WORK_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/hugegraph-scripts-test.XXXXXX") +DATA_ROOT="${WORK_ROOT}/data" +REPO_ROOT="${WORK_ROOT}/repo" +MOCK_BIN="${WORK_ROOT}/bin" +BACKUP_INFO="${WORK_ROOT}/backup-info.json" +BACKUP_NAME=hugegraph-test-backup + +cleanup() { + rm -rf -- "$WORK_ROOT" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +if [[ -x /opt/homebrew/bin/bash ]]; then + PRODUCT_BASH=/opt/homebrew/bin/bash +elif (( BASH_VERSINFO[0] >= 4 )); then + PRODUCT_BASH=$BASH +else + fail "HugeGraph script tests require Bash 4 or newer" +fi + +mkdir -p "$MOCK_BIN" "$REPO_ROOT" + +cat >"${MOCK_BIN}/datasafed" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +: "${TEST_REPO_ROOT:?TEST_REPO_ROOT is required}" +command_name=$1 +shift +case "$command_name" in + push) + source_path=$1 + destination=$2 + destination=${destination#/} + mkdir -p "${TEST_REPO_ROOT}/$(dirname "$destination")" + if [[ "$source_path" == "-" ]]; then + cat >"${TEST_REPO_ROOT}/${destination}" + else + cp "$source_path" "${TEST_REPO_ROOT}/${destination}" + fi + ;; + pull) + source_path=${1#/} + destination=$2 + if [[ "$destination" == "-" ]]; then + cat "${TEST_REPO_ROOT}/${source_path}" + else + cp "${TEST_REPO_ROOT}/${source_path}" "$destination" + fi + ;; + stat) + printf 'TotalSize 4096\n' + ;; + *) + echo "unsupported datasafed command: ${command_name}" >&2 + exit 2 + ;; +esac +MOCK + +cat >"${MOCK_BIN}/curl" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +: "${DATA_ROOT:?DATA_ROOT is required}" +url=${!#} +graph_name=${url##*/graphs/} +graph_name=${graph_name%%/*} +if [[ "${MOCK_CURL_FAIL_GRAPH:-}" == "$graph_name" ]]; then + exit 22 +fi +config="${DATA_ROOT}/graphs/${graph_name}.properties" +data_path=$(awk -F= '$1 == "rocksdb.data_path" {print substr($0, index($0, "=") + 1); exit}' "$config") +snapshot="${DATA_ROOT}/snapshot_$(basename "$data_path")" +for store in g m s; do + mkdir -p "${snapshot}/${store}" + printf '%s-%s\n' "$graph_name" "$store" >"${snapshot}/${store}/CURRENT" +done +MOCK + +chmod +x "${MOCK_BIN}/datasafed" "${MOCK_BIN}/curl" + +write_graph_config() { + local root=$1 + local graph=$2 + local suffix=$3 + + mkdir -p "${root}/graphs" + cat >"${root}/graphs/${graph}.properties" <"${DATA_ROOT}/rocksdb/g/CURRENT" +if run_restore >/dev/null 2>&1; then + fail "idempotent restore accepted corrupted completed data" +fi + +rm -f -- "${DATA_ROOT}/.kb-restored-backup" +printf '%s\n' "$BACKUP_NAME" >"${DATA_ROOT}/.kb-restore-in-progress" +run_restore +[[ "$(<"${DATA_ROOT}/rocksdb/g/CURRENT")" == "hugegraph-g" ]] || fail "retry did not replace partial checkpoint data" +[[ "$(<"${DATA_ROOT}/rocksdb_analytics/g/CURRENT")" == "analytics-g" ]] || fail "retry lost the second graph checkpoint" + +FAIL_DATA_ROOT="${WORK_ROOT}/failed-backup-data" +write_graph_config "$FAIL_DATA_ROOT" hugegraph "" +write_graph_config "$FAIL_DATA_ROOT" analytics "_analytics" +if env \ + PATH="${MOCK_BIN}:${PATH}" \ + TEST_REPO_ROOT="$REPO_ROOT" \ + DATA_ROOT="$FAIL_DATA_ROOT" \ + MOCK_CURL_FAIL_GRAPH=hugegraph \ + DP_DATASAFED_BIN_PATH="$MOCK_BIN" \ + DP_BACKUP_BASE_PATH=/test-failure \ + DP_BACKUP_NAME=failed-backup \ + DP_BACKUP_INFO_FILE="${WORK_ROOT}/failed-backup-info.json" \ + DP_DB_HOST=hugegraph \ + DP_DB_USER=admin \ + DP_DB_PASSWORD=secret \ + "$PRODUCT_BASH" "${ADDON_DIR}/scripts/backup.sh" >/dev/null 2>&1; then + fail "backup unexpectedly succeeded after checkpoint API failure" +fi +[[ -z "$(find "$FAIL_DATA_ROOT" -maxdepth 1 -type d -name 'snapshot_*' -print -quit)" ]] || fail "failed backup left checkpoint directories behind" + +PREEXIST_DATA_ROOT="${WORK_ROOT}/preexisting-backup-data" +write_graph_config "$PREEXIST_DATA_ROOT" hugegraph "" +mkdir -p "${PREEXIST_DATA_ROOT}/snapshot_manual" +printf 'preserve\n' >"${PREEXIST_DATA_ROOT}/snapshot_manual/CURRENT" +if env \ + PATH="${MOCK_BIN}:${PATH}" \ + TEST_REPO_ROOT="$REPO_ROOT" \ + DATA_ROOT="$PREEXIST_DATA_ROOT" \ + DP_DATASAFED_BIN_PATH="$MOCK_BIN" \ + DP_BACKUP_BASE_PATH=/test-preexisting \ + DP_BACKUP_NAME=preexisting-backup \ + DP_BACKUP_INFO_FILE="${WORK_ROOT}/preexisting-backup-info.json" \ + DP_DB_HOST=hugegraph \ + DP_DB_USER=admin \ + DP_DB_PASSWORD=secret \ + "$PRODUCT_BASH" "${ADDON_DIR}/scripts/backup.sh" >/dev/null 2>&1; then + fail "backup unexpectedly accepted a pre-existing checkpoint" +fi +[[ "$(<"${PREEXIST_DATA_ROOT}/snapshot_manual/CURRENT")" == "preserve" ]] || fail "backup removed a pre-existing checkpoint" + +echo "HugeGraph backup/restore script tests passed" diff --git a/addons/hugegraph/values.yaml b/addons/hugegraph/values.yaml new file mode 100644 index 0000000000..964367a48c --- /dev/null +++ b/addons/hugegraph/values.yaml @@ -0,0 +1,14 @@ +nameOverride: "" +fullnameOverride: "" + +image: + registry: docker.io + repository: hugegraph/hugegraph + tag: "1.7.0" + pullPolicy: IfNotPresent + +serviceVersion: "1.7.0" +dataMountPath: /hugegraph-data + +logConfigs: + running: /hugegraph-server/logs/hugegraph-server.log diff --git a/examples/hugegraph/README.md b/examples/hugegraph/README.md new file mode 100644 index 0000000000..2ea441b18d --- /dev/null +++ b/examples/hugegraph/README.md @@ -0,0 +1,37 @@ +# HugeGraph Examples + +These examples target KubeBlocks `release-1.0` and HugeGraph 1.7.0 standalone. + +## Create + +Apply `cluster.yaml`. The Cluster exposes HTTP on port 8080 and Gremlin on +port 8182. The KubeBlocks system account is `admin`. + +## Backup + +Configure a working BackupRepo, then apply `backup.yaml`. The generated backup +policy name is `hugegraph-cluster-server-backup-policy` and the method is +`checkpoint`. + +The backup covers all graph configurations in `/hugegraph-data/graphs` and all +RocksDB checkpoint directories. It does not use a VolumeSnapshotClass. + +## Restore + +Wait until `hugegraph-cluster-backup` is `Completed`, then apply `restore.yaml`. +The target must be a new one-replica HugeGraph 1.7.0 standalone Cluster with a +`data` volume at least as large as the source. + +After restore, verify the target Cluster is Running, authenticate with its +`admin` Secret, list every expected graph, read the source data, and perform a +new write. Backup completion alone does not prove restore. + +## Day-2 + +- `restart.yaml`: restart the server component. +- `stop.yaml` and `start.yaml`: stop and start the server component. +- `verticalscale.yaml`: change CPU and memory. +- `volumeexpand.yaml`: expand the `data` PVC when supported by the StorageClass. + +Horizontal scaling, reconfigure, TLS, switchover, PITR, incremental backup, +single-graph restore, and RebuildInstance are not supported in this version. diff --git a/examples/hugegraph/backup.yaml b/examples/hugegraph/backup.yaml new file mode 100644 index 0000000000..fb95f0724a --- /dev/null +++ b/examples/hugegraph/backup.yaml @@ -0,0 +1,9 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: Backup +metadata: + name: hugegraph-cluster-backup + namespace: demo +spec: + backupMethod: checkpoint + backupPolicyName: hugegraph-cluster-server-backup-policy + deletionPolicy: Delete diff --git a/examples/hugegraph/cluster.yaml b/examples/hugegraph/cluster.yaml new file mode 100644 index 0000000000..922b046191 --- /dev/null +++ b/examples/hugegraph/cluster.yaml @@ -0,0 +1,29 @@ +apiVersion: apps.kubeblocks.io/v1 +kind: Cluster +metadata: + name: hugegraph-cluster + namespace: demo +spec: + clusterDef: hugegraph + topology: standalone + terminationPolicy: Delete + componentSpecs: + - name: server + serviceVersion: "1.7.0" + replicas: 1 + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "500m" + memory: 1Gi + volumeClaimTemplates: + - name: data + spec: + storageClassName: "" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/examples/hugegraph/restart.yaml b/examples/hugegraph/restart.yaml new file mode 100644 index 0000000000..43d4abeb74 --- /dev/null +++ b/examples/hugegraph/restart.yaml @@ -0,0 +1,10 @@ +apiVersion: operations.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: hugegraph-restart + namespace: demo +spec: + clusterName: hugegraph-cluster + type: Restart + restart: + - componentName: server diff --git a/examples/hugegraph/restore.yaml b/examples/hugegraph/restore.yaml new file mode 100644 index 0000000000..d32173aed3 --- /dev/null +++ b/examples/hugegraph/restore.yaml @@ -0,0 +1,31 @@ +apiVersion: apps.kubeblocks.io/v1 +kind: Cluster +metadata: + name: hugegraph-restore + namespace: demo + annotations: + kubeblocks.io/restore-from-backup: '{"server":{"name":"hugegraph-cluster-backup","namespace":"demo","volumeRestorePolicy":"Serial"}}' +spec: + clusterDef: hugegraph + topology: standalone + terminationPolicy: Delete + componentSpecs: + - name: server + serviceVersion: "1.7.0" + replicas: 1 + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "500m" + memory: 1Gi + volumeClaimTemplates: + - name: data + spec: + storageClassName: "" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/examples/hugegraph/start.yaml b/examples/hugegraph/start.yaml new file mode 100644 index 0000000000..e0f9e523b1 --- /dev/null +++ b/examples/hugegraph/start.yaml @@ -0,0 +1,10 @@ +apiVersion: operations.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: hugegraph-start + namespace: demo +spec: + clusterName: hugegraph-cluster + type: Start + start: + - componentName: server diff --git a/examples/hugegraph/stop.yaml b/examples/hugegraph/stop.yaml new file mode 100644 index 0000000000..5c4a3a4476 --- /dev/null +++ b/examples/hugegraph/stop.yaml @@ -0,0 +1,10 @@ +apiVersion: operations.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: hugegraph-stop + namespace: demo +spec: + clusterName: hugegraph-cluster + type: Stop + stop: + - componentName: server diff --git a/examples/hugegraph/verticalscale.yaml b/examples/hugegraph/verticalscale.yaml new file mode 100644 index 0000000000..0ef06b1c85 --- /dev/null +++ b/examples/hugegraph/verticalscale.yaml @@ -0,0 +1,16 @@ +apiVersion: operations.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: hugegraph-verticalscale + namespace: demo +spec: + clusterName: hugegraph-cluster + type: VerticalScaling + verticalScaling: + - componentName: server + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "2" + memory: 4Gi diff --git a/examples/hugegraph/volumeexpand.yaml b/examples/hugegraph/volumeexpand.yaml new file mode 100644 index 0000000000..5020c43d1e --- /dev/null +++ b/examples/hugegraph/volumeexpand.yaml @@ -0,0 +1,13 @@ +apiVersion: operations.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: hugegraph-volumeexpand + namespace: demo +spec: + clusterName: hugegraph-cluster + type: VolumeExpansion + volumeExpansion: + - componentName: server + volumeClaimTemplates: + - name: data + storage: 40Gi From 8a62a1d06ec82db99c0fd7e878fd464b4acf127a Mon Sep 17 00:00:00 2001 From: Amber Date: Mon, 10 Aug 2026 17:08:33 +0800 Subject: [PATCH 02/16] fix: preserve HugeGraph auth across pod replacement --- addons/hugegraph/DESIGN.zh.md | 6 ++++++ addons/hugegraph/README.md | 5 +++++ addons/hugegraph/scripts/start.sh | 5 ++++- addons/hugegraph/templates/cmpd.yaml | 6 ++++++ addons/hugegraph/tests/contract_test.sh | 7 +++++++ 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/addons/hugegraph/DESIGN.zh.md b/addons/hugegraph/DESIGN.zh.md index 2f156dd4cf..4a2ac890fd 100644 --- a/addons/hugegraph/DESIGN.zh.md +++ b/addons/hugegraph/DESIGN.zh.md @@ -62,6 +62,12 @@ PVC 固定挂载到 `/hugegraph-data`: 3. 校验每个 graph 都使用 RocksDB,且 data/WAL 路径是 PVC 根目录的直接子目录。 4. 为每个 graph 启用 `HugeFactoryAuthProxy`。 5. 把上游 `docker/` 初始化标记目录链接到 PVC。 +6. 每次启动都执行上游 `enable-auth.sh`,重建替换容器内的临时认证配置;持久化 + 初始化标记只用于跳过 store 初始化,不能代替该配置。 + +容器通过 Kubernetes `preStop` 调用 `stop-hugegraph.sh`,并保留 30 秒终止窗口。 +KubeBlocks 1.0 的 Stop 不执行 `preTerminate`,因此正常 Stop/Restart 的 RocksDB 收尾 +必须放在 Pod lifecycle hook。 动态多图推荐使用 `clone_graph_name=hugegraph` 创建。HugeGraph 的 RocksDB provider 会为 clone graph 自动生成独立的 `rocksdb_` 和 `wal_` 路径。直接 diff --git a/addons/hugegraph/README.md b/addons/hugegraph/README.md index 247556f606..d168df4386 100644 --- a/addons/hugegraph/README.md +++ b/addons/hugegraph/README.md @@ -38,6 +38,11 @@ configuration files, RocksDB directories, WAL directories, and the upstream initialization marker on this volume. It does not mount a PVC over `/hugegraph-server`. +On every Pod start, the addon rebuilds HugeGraph's authentication settings in +the replacement container before invoking the upstream entrypoint. A `preStop` +hook runs HugeGraph's shutdown script within a 30-second termination window so +Stop, Restart, and pod replacement close RocksDB cleanly. + For additional graphs, use the HugeGraph clone API so the RocksDB provider creates unique persistent paths: diff --git a/addons/hugegraph/scripts/start.sh b/addons/hugegraph/scripts/start.sh index f2ce089ea8..2c62609842 100644 --- a/addons/hugegraph/scripts/start.sh +++ b/addons/hugegraph/scripts/start.sh @@ -125,6 +125,9 @@ if [[ -e "$docker_marker" && ! -L "$docker_marker" ]]; then fi ln -sfn "${DATA_ROOT}/docker" "$docker_marker" -log "starting HugeGraph with ${#graph_configs[@]} persistent graph configuration(s)" cd "$SERVER_HOME" +# The 1.7.0 entrypoint skips enable-auth.sh when the persistent init marker +# exists, but its authentication config lives in the replacement container. +./bin/enable-auth.sh +log "starting HugeGraph with ${#graph_configs[@]} persistent graph configuration(s)" exec /usr/bin/dumb-init -- ./docker-entrypoint.sh diff --git a/addons/hugegraph/templates/cmpd.yaml b/addons/hugegraph/templates/cmpd.yaml index 9d258d87dd..88dc77f40d 100644 --- a/addons/hugegraph/templates/cmpd.yaml +++ b/addons/hugegraph/templates/cmpd.yaml @@ -61,6 +61,7 @@ spec: - name: running filePathPattern: {{ .Values.logConfigs.running }} runtime: + terminationGracePeriodSeconds: 30 containers: - name: hugegraph imagePullPolicy: {{ .Values.image.pullPolicy }} @@ -70,6 +71,11 @@ spec: command: - /bin/bash - /scripts/start.sh + lifecycle: + preStop: + exec: + command: + - /hugegraph-server/bin/stop-hugegraph.sh env: - name: PASSWORD value: $(ADMIN_PASSWORD) diff --git a/addons/hugegraph/tests/contract_test.sh b/addons/hugegraph/tests/contract_test.sh index 4f816ec16b..f89d618811 100644 --- a/addons/hugegraph/tests/contract_test.sh +++ b/addons/hugegraph/tests/contract_test.sh @@ -72,6 +72,10 @@ assert_contains "${ADDON_DIR}/templates/backuppolicytemplate.yaml" 'name: checkp assert_contains "${ADDON_DIR}/templates/backuppolicytemplate.yaml" 'account: admin' assert_contains "${ADDON_DIR}/templates/actionset.yaml" 'prepareData:' assert_contains "${ADDON_DIR}/templates/actionset.yaml" 'runOnTargetPodNode: true' +assert_contains "${ADDON_DIR}/scripts/start.sh" './bin/enable-auth\.sh' +assert_contains "${ADDON_DIR}/templates/cmpd.yaml" 'terminationGracePeriodSeconds: 30' +assert_contains "${ADDON_DIR}/templates/cmpd.yaml" 'preStop:' +assert_contains "${ADDON_DIR}/templates/cmpd.yaml" '/hugegraph-server/bin/stop-hugegraph\.sh' assert_not_contains_tree 'volume[-_ ]snapshot' assert_not_contains_tree 'snapshotVolumes:[[:space:]]*true' @@ -97,6 +101,9 @@ assert_contains "${definition_render}" 'image: docker.io/hugegraph/hugegraph:1.7 assert_contains "${definition_render}" 'mountPath: /hugegraph-data' assert_contains "${definition_render}" 'containerPort: 8080' assert_contains "${definition_render}" 'containerPort: 8182' +assert_contains "${definition_render}" 'terminationGracePeriodSeconds: 30' +assert_contains "${definition_render}" 'preStop:' +assert_contains "${definition_render}" '/hugegraph-server/bin/stop-hugegraph\.sh' assert_contains "${cluster_render}" 'clusterDef: hugegraph' assert_contains "${cluster_render}" 'topology: standalone' assert_contains "${cluster_render}" 'replicas: 1' From f0411fc8c2a9d17b4f4579ecc6dee731ce8712c0 Mon Sep 17 00:00:00 2001 From: Amber Date: Mon, 10 Aug 2026 17:17:33 +0800 Subject: [PATCH 03/16] test: make HugeGraph shutdown auditable --- addons/hugegraph/DESIGN.zh.md | 7 ++-- addons/hugegraph/README.md | 3 +- addons/hugegraph/scripts/shutdown.sh | 27 ++++++++++++++ addons/hugegraph/templates/cmpd.yaml | 3 +- .../hugegraph/templates/script-template.yaml | 2 + addons/hugegraph/tests/contract_test.sh | 9 ++++- addons/hugegraph/tests/scripts_test.sh | 37 +++++++++++++++++++ 7 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 addons/hugegraph/scripts/shutdown.sh diff --git a/addons/hugegraph/DESIGN.zh.md b/addons/hugegraph/DESIGN.zh.md index 4a2ac890fd..95cb7e8f04 100644 --- a/addons/hugegraph/DESIGN.zh.md +++ b/addons/hugegraph/DESIGN.zh.md @@ -65,9 +65,10 @@ PVC 固定挂载到 `/hugegraph-data`: 6. 每次启动都执行上游 `enable-auth.sh`,重建替换容器内的临时认证配置;持久化 初始化标记只用于跳过 store 初始化,不能代替该配置。 -容器通过 Kubernetes `preStop` 调用 `stop-hugegraph.sh`,并保留 30 秒终止窗口。 -KubeBlocks 1.0 的 Stop 不执行 `preTerminate`,因此正常 Stop/Restart 的 RocksDB 收尾 -必须放在 Pod lifecycle hook。 +容器通过 Kubernetes `preStop` 调用 `shutdown.sh`,由它执行上游 +`stop-hugegraph.sh`,并把开始/完成记录写到 PVC 上的 `.kb-prestop.log`;Pod 保留 30 +秒终止窗口。KubeBlocks 1.0 的 Stop 不执行 `preTerminate`,因此正常 Stop/Restart 的 +RocksDB 收尾必须放在 Pod lifecycle hook。 动态多图推荐使用 `clone_graph_name=hugegraph` 创建。HugeGraph 的 RocksDB provider 会为 clone graph 自动生成独立的 `rocksdb_` 和 `wal_` 路径。直接 diff --git a/addons/hugegraph/README.md b/addons/hugegraph/README.md index d168df4386..3121d6765a 100644 --- a/addons/hugegraph/README.md +++ b/addons/hugegraph/README.md @@ -41,7 +41,8 @@ initialization marker on this volume. It does not mount a PVC over On every Pod start, the addon rebuilds HugeGraph's authentication settings in the replacement container before invoking the upstream entrypoint. A `preStop` hook runs HugeGraph's shutdown script within a 30-second termination window so -Stop, Restart, and pod replacement close RocksDB cleanly. +Stop, Restart, and pod replacement close RocksDB cleanly. The hook records its +start and completion in `/hugegraph-data/.kb-prestop.log` for diagnosis. For additional graphs, use the HugeGraph clone API so the RocksDB provider creates unique persistent paths: diff --git a/addons/hugegraph/scripts/shutdown.sh b/addons/hugegraph/scripts/shutdown.sh new file mode 100644 index 0000000000..2833addf91 --- /dev/null +++ b/addons/hugegraph/scripts/shutdown.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +SERVER_HOME=${HUGEGRAPH_SERVER_HOME:-/hugegraph-server} +DATA_ROOT=${HUGEGRAPH_DATA_ROOT:-/hugegraph-data} +AUDIT_LOG="${DATA_ROOT}/.kb-prestop.log" + +log() { + local message + message="$(date -u +%Y-%m-%dT%H:%M:%SZ) INFO: $*" + printf '%s\n' "$message" + printf '%s\n' "$message" >>"$AUDIT_LOG" +} + +cd "$SERVER_HOME" +pid=$(cat ./bin/pid 2>/dev/null || true) + +if [[ "$pid" =~ ^[0-9]+$ ]] && kill -0 "$pid" 2>/dev/null; then + log "graceful shutdown started for HugeGraphServer pid ${pid}" + ./bin/stop-hugegraph.sh +else + log "HugeGraphServer is already stopped; skipping shutdown command" +fi + +sync +log "graceful shutdown completed" diff --git a/addons/hugegraph/templates/cmpd.yaml b/addons/hugegraph/templates/cmpd.yaml index 88dc77f40d..d390d213ef 100644 --- a/addons/hugegraph/templates/cmpd.yaml +++ b/addons/hugegraph/templates/cmpd.yaml @@ -75,7 +75,8 @@ spec: preStop: exec: command: - - /hugegraph-server/bin/stop-hugegraph.sh + - /bin/bash + - /scripts/shutdown.sh env: - name: PASSWORD value: $(ADMIN_PASSWORD) diff --git a/addons/hugegraph/templates/script-template.yaml b/addons/hugegraph/templates/script-template.yaml index 19a7826599..4216c3be73 100644 --- a/addons/hugegraph/templates/script-template.yaml +++ b/addons/hugegraph/templates/script-template.yaml @@ -7,3 +7,5 @@ metadata: data: start.sh: |- {{- .Files.Get "scripts/start.sh" | nindent 4 }} + shutdown.sh: |- + {{- .Files.Get "scripts/shutdown.sh" | nindent 4 }} diff --git a/addons/hugegraph/tests/contract_test.sh b/addons/hugegraph/tests/contract_test.sh index f89d618811..e83bd693c7 100644 --- a/addons/hugegraph/tests/contract_test.sh +++ b/addons/hugegraph/tests/contract_test.sh @@ -41,6 +41,7 @@ required_files=( "${ADDON_DIR}/templates/actionset.yaml" "${ADDON_DIR}/templates/script-template.yaml" "${ADDON_DIR}/scripts/start.sh" + "${ADDON_DIR}/scripts/shutdown.sh" "${ADDON_DIR}/scripts/backup.sh" "${ADDON_DIR}/scripts/restore.sh" "${ADDON_DIR}/tests/scripts_test.sh" @@ -57,6 +58,7 @@ done bash -n \ "${ADDON_DIR}/scripts/start.sh" \ + "${ADDON_DIR}/scripts/shutdown.sh" \ "${ADDON_DIR}/scripts/backup.sh" \ "${ADDON_DIR}/scripts/restore.sh" @@ -73,9 +75,12 @@ assert_contains "${ADDON_DIR}/templates/backuppolicytemplate.yaml" 'account: adm assert_contains "${ADDON_DIR}/templates/actionset.yaml" 'prepareData:' assert_contains "${ADDON_DIR}/templates/actionset.yaml" 'runOnTargetPodNode: true' assert_contains "${ADDON_DIR}/scripts/start.sh" './bin/enable-auth\.sh' +assert_contains "${ADDON_DIR}/scripts/shutdown.sh" 'graceful shutdown started' +assert_contains "${ADDON_DIR}/scripts/shutdown.sh" 'graceful shutdown completed' +assert_contains "${ADDON_DIR}/scripts/shutdown.sh" '\.kb-prestop\.log' assert_contains "${ADDON_DIR}/templates/cmpd.yaml" 'terminationGracePeriodSeconds: 30' assert_contains "${ADDON_DIR}/templates/cmpd.yaml" 'preStop:' -assert_contains "${ADDON_DIR}/templates/cmpd.yaml" '/hugegraph-server/bin/stop-hugegraph\.sh' +assert_contains "${ADDON_DIR}/templates/cmpd.yaml" '/scripts/shutdown\.sh' assert_not_contains_tree 'volume[-_ ]snapshot' assert_not_contains_tree 'snapshotVolumes:[[:space:]]*true' @@ -103,7 +108,7 @@ assert_contains "${definition_render}" 'containerPort: 8080' assert_contains "${definition_render}" 'containerPort: 8182' assert_contains "${definition_render}" 'terminationGracePeriodSeconds: 30' assert_contains "${definition_render}" 'preStop:' -assert_contains "${definition_render}" '/hugegraph-server/bin/stop-hugegraph\.sh' +assert_contains "${definition_render}" '/scripts/shutdown\.sh' assert_contains "${cluster_render}" 'clusterDef: hugegraph' assert_contains "${cluster_render}" 'topology: standalone' assert_contains "${cluster_render}" 'replicas: 1' diff --git a/addons/hugegraph/tests/scripts_test.sh b/addons/hugegraph/tests/scripts_test.sh index be0d33f55b..57da8d1f6f 100755 --- a/addons/hugegraph/tests/scripts_test.sh +++ b/addons/hugegraph/tests/scripts_test.sh @@ -89,6 +89,43 @@ MOCK chmod +x "${MOCK_BIN}/datasafed" "${MOCK_BIN}/curl" +SHUTDOWN_SERVER_HOME="${WORK_ROOT}/shutdown-server" +SHUTDOWN_DATA_ROOT="${WORK_ROOT}/shutdown-data" +mkdir -p "${SHUTDOWN_SERVER_HOME}/bin" "$SHUTDOWN_DATA_ROOT" +cat >"${SHUTDOWN_SERVER_HOME}/bin/stop-hugegraph.sh" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +pid=$(cat "$(dirname "$0")/pid") +kill -TERM "$pid" +wait "$pid" 2>/dev/null || true +rm -f "$(dirname "$0")/pid" +MOCK +chmod +x "${SHUTDOWN_SERVER_HOME}/bin/stop-hugegraph.sh" + +sleep 300 & +shutdown_pid=$! +printf '%s\n' "$shutdown_pid" >"${SHUTDOWN_SERVER_HOME}/bin/pid" +env \ + HUGEGRAPH_SERVER_HOME="$SHUTDOWN_SERVER_HOME" \ + HUGEGRAPH_DATA_ROOT="$SHUTDOWN_DATA_ROOT" \ + "$PRODUCT_BASH" "${ADDON_DIR}/scripts/shutdown.sh" +wait "$shutdown_pid" 2>/dev/null || true +if kill -0 "$shutdown_pid" 2>/dev/null; then + kill "$shutdown_pid" 2>/dev/null || true + fail "shutdown script left the HugeGraph process running" +fi +grep -Fq 'graceful shutdown started' "${SHUTDOWN_DATA_ROOT}/.kb-prestop.log" \ + || fail "shutdown audit log missed the start record" +grep -Fq 'graceful shutdown completed' "${SHUTDOWN_DATA_ROOT}/.kb-prestop.log" \ + || fail "shutdown audit log missed the completion record" + +env \ + HUGEGRAPH_SERVER_HOME="$SHUTDOWN_SERVER_HOME" \ + HUGEGRAPH_DATA_ROOT="$SHUTDOWN_DATA_ROOT" \ + "$PRODUCT_BASH" "${ADDON_DIR}/scripts/shutdown.sh" +grep -Fq 'already stopped' "${SHUTDOWN_DATA_ROOT}/.kb-prestop.log" \ + || fail "shutdown retry did not record the already-stopped state" + write_graph_config() { local root=$1 local graph=$2 From 3634d7b84c305ce076457ccb450faa9b79b76ea8 Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Wed, 19 Aug 2026 01:36:04 +0800 Subject: [PATCH 04/16] feat(hugegraph): add native metrics exporter sidecar (#3413) Co-authored-by: Ava Co-authored-by: Amber --- addons/hugegraph/.helmignore | 1 + addons/hugegraph/exporter/Dockerfile | 16 + .../exporter/cmd/hugegraph-exporter/main.go | 55 +++ addons/hugegraph/exporter/exporter.go | 345 ++++++++++++++ addons/hugegraph/exporter/exporter_test.go | 446 ++++++++++++++++++ addons/hugegraph/exporter/go.mod | 20 + addons/hugegraph/exporter/go.sum | 24 + addons/hugegraph/exporter/parser.go | 389 +++++++++++++++ addons/hugegraph/templates/_helpers.tpl | 4 + addons/hugegraph/templates/cmpd.yaml | 32 ++ .../templates/monitordefinition.yaml | 21 + addons/hugegraph/tests/contract_test.sh | 78 ++- addons/hugegraph/values.yaml | 14 + 13 files changed, 1444 insertions(+), 1 deletion(-) create mode 100644 addons/hugegraph/exporter/Dockerfile create mode 100644 addons/hugegraph/exporter/cmd/hugegraph-exporter/main.go create mode 100644 addons/hugegraph/exporter/exporter.go create mode 100644 addons/hugegraph/exporter/exporter_test.go create mode 100644 addons/hugegraph/exporter/go.mod create mode 100644 addons/hugegraph/exporter/go.sum create mode 100644 addons/hugegraph/exporter/parser.go create mode 100644 addons/hugegraph/templates/monitordefinition.yaml diff --git a/addons/hugegraph/.helmignore b/addons/hugegraph/.helmignore index 867c3249d6..217ff618c7 100644 --- a/addons/hugegraph/.helmignore +++ b/addons/hugegraph/.helmignore @@ -4,3 +4,4 @@ tests/ *.swp *.tmp +exporter/ diff --git a/addons/hugegraph/exporter/Dockerfile b/addons/hugegraph/exporter/Dockerfile new file mode 100644 index 0000000000..1b64d2602d --- /dev/null +++ b/addons/hugegraph/exporter/Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.22.12-alpine3.21 AS builder + +ARG TARGETOS=linux +ARG TARGETARCH + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -ldflags="-s -w" -o /out/hugegraph-exporter ./cmd/hugegraph-exporter + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=builder /out/hugegraph-exporter /hugegraph-exporter +EXPOSE 9404 +ENTRYPOINT ["/hugegraph-exporter"] diff --git a/addons/hugegraph/exporter/cmd/hugegraph-exporter/main.go b/addons/hugegraph/exporter/cmd/hugegraph-exporter/main.go new file mode 100644 index 0000000000..f5f892e490 --- /dev/null +++ b/addons/hugegraph/exporter/cmd/hugegraph-exporter/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "errors" + "log" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/apecloud/kubeblocks-addons/addons/hugegraph/exporter" +) + +func main() { + logger := log.New(os.Stderr, "hugegraph-exporter: ", log.LstdFlags) + config := exporter.DefaultConfig() + config.Username = os.Getenv("HUGEGRAPH_USERNAME") + config.Password = os.Getenv("HUGEGRAPH_PASSWORD") + metricsExporter, err := exporter.New(config, logger) + if err != nil { + logger.Fatal("invalid exporter configuration") + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + go metricsExporter.Run(ctx) + + mux := http.NewServeMux() + mux.Handle("/metrics", metricsExporter.MetricsHandler()) + mux.HandleFunc("/healthz", func(response http.ResponseWriter, _ *http.Request) { + response.WriteHeader(http.StatusOK) + }) + server := &http.Server{ + Addr: ":9404", + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 30 * time.Second, + } + + go func() { + <-ctx.Done() + shutdownContext, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = server.Shutdown(shutdownContext) + }() + + logger.Printf("serving metrics on %s", server.Addr) + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Fatal("metrics server stopped unexpectedly") + } +} diff --git a/addons/hugegraph/exporter/exporter.go b/addons/hugegraph/exporter/exporter.go new file mode 100644 index 0000000000..9a1993ea2d --- /dev/null +++ b/addons/hugegraph/exporter/exporter.go @@ -0,0 +1,345 @@ +package exporter + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +const ( + defaultEndpoint = "http://127.0.0.1:8080/metrics?type=json" + defaultCollectionInterval = 60 * time.Second + defaultRequestTimeout = 5 * time.Second + defaultMaxBodyBytes = int64(1 << 20) + defaultMaxRoutes = 128 + defaultMaxCaches = 64 +) + +type Config struct { + Endpoint string + Username string + Password string + CollectionInterval time.Duration + RequestTimeout time.Duration + MaxBodyBytes int64 + MaxRoutes int + MaxCaches int + Now func() time.Time +} + +func DefaultConfig() Config { + return Config{ + Endpoint: defaultEndpoint, + CollectionInterval: defaultCollectionInterval, + RequestTimeout: defaultRequestTimeout, + MaxBodyBytes: defaultMaxBodyBytes, + MaxRoutes: defaultMaxRoutes, + MaxCaches: defaultMaxCaches, + Now: time.Now, + } +} + +type metricSample struct { + desc *prometheus.Desc + valueType prometheus.ValueType + value float64 + labels []string +} + +type collectionState struct { + samples []metricSample + up float64 + stale float64 + lastSuccess float64 + errors map[string]uint64 +} + +type Exporter struct { + config Config + client *http.Client + logger *log.Logger + + stateMu sync.RWMutex + state collectionState + + flightMu sync.Mutex + inflight chan struct{} + lastErr error +} + +func New(config Config, logger *log.Logger) (*Exporter, error) { + if err := normalizeConfig(&config); err != nil { + return nil, err + } + if logger == nil { + logger = log.New(io.Discard, "", 0) + } + return &Exporter{ + config: config, + client: &http.Client{Timeout: config.RequestTimeout}, + logger: logger, + state: collectionState{ + errors: make(map[string]uint64), + }, + }, nil +} + +func normalizeConfig(config *Config) error { + if config.Endpoint == "" { + config.Endpoint = defaultEndpoint + } + parsed, err := url.Parse(config.Endpoint) + if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") { + return errors.New("invalid metrics endpoint") + } + if parsed.User != nil { + return errors.New("metrics endpoint must not contain user information") + } + if config.Username == "" || config.Password == "" { + return errors.New("metrics credentials are required") + } + if config.CollectionInterval <= 0 { + config.CollectionInterval = defaultCollectionInterval + } + if config.RequestTimeout <= 0 { + config.RequestTimeout = defaultRequestTimeout + } + if config.MaxBodyBytes <= 0 { + config.MaxBodyBytes = defaultMaxBodyBytes + } + if config.MaxRoutes <= 0 { + config.MaxRoutes = defaultMaxRoutes + } + if config.MaxCaches <= 0 { + config.MaxCaches = defaultMaxCaches + } + if config.Now == nil { + config.Now = time.Now + } + return nil +} + +func (exporter *Exporter) Refresh(ctx context.Context) error { + exporter.flightMu.Lock() + if exporter.inflight != nil { + inflight := exporter.inflight + exporter.flightMu.Unlock() + select { + case <-inflight: + exporter.flightMu.Lock() + err := exporter.lastErr + exporter.flightMu.Unlock() + return err + case <-ctx.Done(): + return ctx.Err() + } + } + exporter.inflight = make(chan struct{}) + inflight := exporter.inflight + exporter.flightMu.Unlock() + + err := exporter.refreshOnce(ctx) + + exporter.flightMu.Lock() + exporter.lastErr = err + close(inflight) + exporter.inflight = nil + exporter.flightMu.Unlock() + return err +} + +func (exporter *Exporter) refreshOnce(ctx context.Context) error { + requestContext, cancel := context.WithTimeout(ctx, exporter.config.RequestTimeout) + defer cancel() + + request, err := http.NewRequestWithContext(requestContext, http.MethodGet, exporter.config.Endpoint, nil) + if err != nil { + return exporter.recordFailure("request") + } + request.SetBasicAuth(exporter.config.Username, exporter.config.Password) + response, err := exporter.client.Do(request) + if err != nil { + if errors.Is(requestContext.Err(), context.DeadlineExceeded) { + return exporter.recordFailure("timeout") + } + return exporter.recordFailure("connect") + } + defer response.Body.Close() + + if response.StatusCode == http.StatusUnauthorized || response.StatusCode == http.StatusForbidden { + return exporter.recordFailure("auth") + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return exporter.recordFailure("status") + } + + body, err := io.ReadAll(io.LimitReader(response.Body, exporter.config.MaxBodyBytes+1)) + if err != nil { + return exporter.recordFailure("read") + } + if int64(len(body)) > exporter.config.MaxBodyBytes { + return exporter.recordFailure("body_size") + } + + samples, parseErr := parsePayload(body, exporter.config.MaxRoutes, exporter.config.MaxCaches) + if parseErr != nil { + return exporter.recordFailure(parseErr.stage) + } + + exporter.stateMu.Lock() + exporter.state.samples = samples + exporter.state.up = 1 + exporter.state.stale = 0 + exporter.state.lastSuccess = float64(exporter.config.Now().Unix()) + exporter.stateMu.Unlock() + return nil +} + +func (exporter *Exporter) recordFailure(stage string) error { + exporter.stateMu.Lock() + exporter.state.up = 0 + if len(exporter.state.samples) > 0 { + exporter.state.stale = 1 + } + exporter.state.errors[stage]++ + exporter.stateMu.Unlock() + exporter.logger.Printf("collection failed at stage %s", stage) + return fmt.Errorf("collection failed at stage %s", stage) +} + +func (exporter *Exporter) Run(ctx context.Context) { + _ = exporter.Refresh(ctx) + ticker := time.NewTicker(exporter.config.CollectionInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _ = exporter.Refresh(ctx) + } + } +} + +func (exporter *Exporter) MetricsHandler() http.Handler { + registry := prometheus.NewRegistry() + registry.MustRegister(exporter) + return promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) +} + +func (exporter *Exporter) Describe(channel chan<- *prometheus.Desc) { + for _, desc := range allDescs { + channel <- desc + } +} + +func (exporter *Exporter) Collect(channel chan<- prometheus.Metric) { + exporter.stateMu.RLock() + state := collectionState{ + samples: append([]metricSample(nil), exporter.state.samples...), + up: exporter.state.up, + stale: exporter.state.stale, + lastSuccess: exporter.state.lastSuccess, + errors: make(map[string]uint64, len(exporter.state.errors)), + } + for stage, count := range exporter.state.errors { + state.errors[stage] = count + } + exporter.stateMu.RUnlock() + + channel <- prometheus.MustNewConstMetric(collectionUpDesc, prometheus.GaugeValue, state.up) + channel <- prometheus.MustNewConstMetric(collectionStaleDesc, prometheus.GaugeValue, state.stale) + channel <- prometheus.MustNewConstMetric(lastSuccessDesc, prometheus.GaugeValue, state.lastSuccess) + for stage, count := range state.errors { + channel <- prometheus.MustNewConstMetric(collectionErrorsDesc, prometheus.CounterValue, float64(count), stage) + } + for _, sample := range state.samples { + channel <- prometheus.MustNewConstMetric(sample.desc, sample.valueType, sample.value, sample.labels...) + } +} + +var ( + collectionUpDesc = prometheus.NewDesc( + "hugegraph_collection_up", + "Whether the latest HugeGraph JSON collection succeeded.", + nil, nil, + ) + collectionStaleDesc = prometheus.NewDesc( + "hugegraph_collection_stale", + "Whether the exporter is serving the last successful snapshot after a collection failure.", + nil, nil, + ) + lastSuccessDesc = prometheus.NewDesc( + "hugegraph_collection_last_success_unixtime_seconds", + "Unix time of the last successful HugeGraph JSON collection.", + nil, nil, + ) + collectionErrorsDesc = prometheus.NewDesc( + "hugegraph_collection_errors_total", + "HugeGraph JSON collection failures by bounded stage.", + []string{"stage"}, nil, + ) + httpRequestsDesc = prometheus.NewDesc( + "hugegraph_http_requests_total", + "HugeGraph HTTP requests observed by the engine.", + []string{"route", "method", "result"}, nil, + ) + httpResponseTimeDesc = prometheus.NewDesc( + "hugegraph_http_response_time_ms", + "HugeGraph HTTP response time snapshot in milliseconds.", + []string{"route", "method", "stat"}, nil, + ) + transactionCommitsDesc = prometheus.NewDesc( + "hugegraph_transaction_commits_total", + "HugeGraph transaction commit outcomes.", + []string{"result"}, nil, + ) + gremlinErrorsDesc = prometheus.NewDesc( + "hugegraph_gremlin_errors_total", + "HugeGraph Gremlin server errors.", + nil, nil, + ) + cacheEventsDesc = prometheus.NewDesc( + "hugegraph_cache_events_total", + "HugeGraph cache events.", + []string{"graph", "cache", "event"}, nil, + ) + cacheEntriesDesc = prometheus.NewDesc( + "hugegraph_cache_entries", + "HugeGraph cache entries and capacity.", + []string{"graph", "cache", "state"}, nil, + ) + tasksPendingDesc = prometheus.NewDesc( + "hugegraph_tasks_pending", + "HugeGraph pending tasks.", + []string{"pool"}, nil, + ) + taskWorkersDesc = prometheus.NewDesc( + "hugegraph_task_workers", + "HugeGraph task workers.", + []string{"pool", "state"}, nil, + ) + allDescs = []*prometheus.Desc{ + collectionUpDesc, + collectionStaleDesc, + lastSuccessDesc, + collectionErrorsDesc, + httpRequestsDesc, + httpResponseTimeDesc, + transactionCommitsDesc, + gremlinErrorsDesc, + cacheEventsDesc, + cacheEntriesDesc, + tasksPendingDesc, + taskWorkersDesc, + } +) diff --git a/addons/hugegraph/exporter/exporter_test.go b/addons/hugegraph/exporter/exporter_test.go new file mode 100644 index 0000000000..5ea5f87ecd --- /dev/null +++ b/addons/hugegraph/exporter/exporter_test.go @@ -0,0 +1,446 @@ +package exporter + +import ( + "bytes" + "context" + "fmt" + "io" + "log" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/common/expfmt" +) + +const ( + testUsername = "metrics-reader" + testPassword = "do-not-disclose-password" +) + +func TestDefaultConfigLocksCollectionBudget(t *testing.T) { + config := DefaultConfig() + if config.Endpoint != "http://127.0.0.1:8080/metrics?type=json" { + t.Fatalf("unexpected endpoint: %q", config.Endpoint) + } + if config.CollectionInterval != 60*time.Second { + t.Fatalf("unexpected collection interval: %s", config.CollectionInterval) + } + if config.RequestTimeout != 5*time.Second { + t.Fatalf("unexpected request timeout: %s", config.RequestTimeout) + } + if config.MaxBodyBytes != 1<<20 { + t.Fatalf("unexpected body limit: %d", config.MaxBodyBytes) + } +} + +func TestRefreshUsesBasicAuthAndExportsAllowlistedMetrics(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + username, password, ok := request.BasicAuth() + if !ok || username != testUsername || password != testPassword { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte(fullFixture(11))) + })) + t.Cleanup(server.Close) + + exporter := newTestExporter(t, server.URL) + if err := exporter.Refresh(context.Background()); err != nil { + t.Fatalf("refresh failed: %v", err) + } + + expected := ` +# HELP hugegraph_http_requests_total HugeGraph HTTP requests observed by the engine. +# TYPE hugegraph_http_requests_total counter +hugegraph_http_requests_total{method="GET",result="total",route="/graphspaces/DEFAULT/graphs/graph/vertices"} 11 +# HELP hugegraph_http_response_time_ms HugeGraph HTTP response time snapshot in milliseconds. +# TYPE hugegraph_http_response_time_ms gauge +hugegraph_http_response_time_ms{method="GET",route="/graphspaces/DEFAULT/graphs/graph/vertices",stat="max"} 30 +hugegraph_http_response_time_ms{method="GET",route="/graphspaces/DEFAULT/graphs/graph/vertices",stat="mean"} 12.5 +hugegraph_http_response_time_ms{method="GET",route="/graphspaces/DEFAULT/graphs/graph/vertices",stat="p95"} 20 +hugegraph_http_response_time_ms{method="GET",route="/graphspaces/DEFAULT/graphs/graph/vertices",stat="p99"} 28 +# HELP hugegraph_transaction_commits_total HugeGraph transaction commit outcomes. +# TYPE hugegraph_transaction_commits_total counter +hugegraph_transaction_commits_total{result="succeed"} 9 +# HELP hugegraph_gremlin_errors_total HugeGraph Gremlin server errors. +# TYPE hugegraph_gremlin_errors_total counter +hugegraph_gremlin_errors_total 2 +# HELP hugegraph_tasks_pending HugeGraph pending tasks. +# TYPE hugegraph_tasks_pending gauge +hugegraph_tasks_pending{pool="task"} 3 +# HELP hugegraph_task_workers HugeGraph task workers. +# TYPE hugegraph_task_workers gauge +hugegraph_task_workers{pool="task",state="configured"} 4 +# HELP hugegraph_cache_events_total HugeGraph cache events. +# TYPE hugegraph_cache_events_total counter +hugegraph_cache_events_total{cache="vertices",event="hit",graph="DEFAULT/hugegraph"} 7 +# HELP hugegraph_cache_entries HugeGraph cache entries and capacity. +# TYPE hugegraph_cache_entries gauge +hugegraph_cache_entries{cache="vertices",graph="DEFAULT/hugegraph",state="capacity"} 100 +` + if err := testutil.CollectAndCompare( + exporter, + strings.NewReader(expected), + "hugegraph_http_requests_total", + "hugegraph_http_response_time_ms", + "hugegraph_transaction_commits_total", + "hugegraph_gremlin_errors_total", + "hugegraph_tasks_pending", + "hugegraph_task_workers", + "hugegraph_cache_events_total", + "hugegraph_cache_entries", + ); err != nil { + t.Fatal(err) + } + + metrics := collectText(t, exporter) + if strings.Contains(metrics, `route="/metrics`) { + t.Fatalf("self traffic leaked into business metrics:\n%s", metrics) + } +} + +func TestRefreshPreservesLastSuccessAndMarksStale(t *testing.T) { + var fail atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if fail.Load() { + http.Error(w, "sensitive response body", http.StatusInternalServerError) + return + } + _, _ = w.Write([]byte(fullFixture(17))) + })) + t.Cleanup(server.Close) + + var logs bytes.Buffer + exporter := newTestExporterWithLogger(t, server.URL, log.New(&logs, "", 0)) + if err := exporter.Refresh(context.Background()); err != nil { + t.Fatalf("initial refresh failed: %v", err) + } + fail.Store(true) + if err := exporter.Refresh(context.Background()); err == nil { + t.Fatal("expected failed refresh") + } + + metrics := collectText(t, exporter) + for _, expected := range []string{ + "hugegraph_collection_up 0", + "hugegraph_collection_stale 1", + "hugegraph_collection_last_success_unixtime_seconds 1.7e+09", + `hugegraph_collection_errors_total{stage="status"} 1`, + `hugegraph_http_requests_total{method="GET",result="total",route="/graphspaces/DEFAULT/graphs/graph/vertices"} 17`, + } { + if !strings.Contains(metrics, expected) { + t.Fatalf("missing %q in metrics:\n%s", expected, metrics) + } + } + for _, secret := range []string{testUsername, testPassword, "sensitive response body", "Basic ", "Authorization"} { + if strings.Contains(logs.String(), secret) { + t.Fatalf("log disclosed %q: %s", secret, logs.String()) + } + } +} + +func TestRefreshIsSingleFlight(t *testing.T) { + var requests atomic.Int32 + entered := make(chan struct{}) + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if requests.Add(1) == 1 { + close(entered) + } + <-release + _, _ = w.Write([]byte(fullFixture(1))) + })) + t.Cleanup(server.Close) + + exporter := newTestExporter(t, server.URL) + const callers = 16 + var waitGroup sync.WaitGroup + waitGroup.Add(callers) + for range callers { + go func() { + defer waitGroup.Done() + if err := exporter.Refresh(context.Background()); err != nil { + t.Errorf("refresh failed: %v", err) + } + }() + } + <-entered + time.Sleep(20 * time.Millisecond) + close(release) + waitGroup.Wait() + if got := requests.Load(); got != 1 { + t.Fatalf("expected one upstream request, got %d", got) + } +} + +func TestMetricsScrapesUseCacheAndProduceValidPrometheusText(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + _, _ = w.Write([]byte(fullFixture(8))) + })) + t.Cleanup(server.Close) + + exporter := newTestExporter(t, server.URL) + if err := exporter.Refresh(context.Background()); err != nil { + t.Fatalf("refresh failed: %v", err) + } + handler := exporter.MetricsHandler() + for range 10 { + request := httptest.NewRequest(http.MethodGet, "/metrics", nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("unexpected scrape status: %d", response.Code) + } + parser := expfmt.TextParser{} + if _, err := parser.TextToMetricFamilies(response.Body); err != nil { + t.Fatalf("invalid Prometheus text: %v", err) + } + } + if got := requests.Load(); got != 1 { + t.Fatalf("scrapes triggered %d upstream requests, want 1", got) + } +} + +func TestRefreshClassifiesAuthDecodeAndSchemaFailures(t *testing.T) { + tests := []struct { + name string + statusCode int + body string + stage string + }{ + {name: "auth", statusCode: http.StatusUnauthorized, body: "do-not-log-auth-body", stage: "auth"}, + {name: "decode", statusCode: http.StatusOK, body: `{not-json}`, stage: "decode"}, + {name: "trailing-json", statusCode: http.StatusOK, body: fullFixture(1) + `{}`, stage: "decode"}, + {name: "schema", statusCode: http.StatusOK, body: `{"gauges":{},"counters":{},"histograms":{},"meters":{}}`, stage: "schema"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(test.statusCode) + _, _ = io.WriteString(w, test.body) + })) + t.Cleanup(server.Close) + + var logs bytes.Buffer + exporter := newTestExporterWithLogger(t, server.URL, log.New(&logs, "", 0)) + if err := exporter.Refresh(context.Background()); err == nil { + t.Fatal("expected refresh failure") + } + metrics := collectText(t, exporter) + expected := fmt.Sprintf(`stage="%s"} 1`, test.stage) + if !strings.Contains(metrics, expected) { + t.Fatalf("missing %q: %s", expected, metrics) + } + if strings.Contains(logs.String(), test.body) { + t.Fatalf("response body leaked to logs: %s", logs.String()) + } + }) + } +} + +func TestConfigRejectsURLUserInfo(t *testing.T) { + config := DefaultConfig() + config.Endpoint = "http://user:password@127.0.0.1:8080/metrics?type=json" + config.Username = testUsername + config.Password = testPassword + if _, err := New(config, log.New(io.Discard, "", 0)); err == nil { + t.Fatal("expected URL user information to be rejected") + } +} + +func TestCounterResetExportsNewRawValue(t *testing.T) { + var count atomic.Int64 + count.Store(100) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, fullFixture(count.Load())) + })) + t.Cleanup(server.Close) + + exporter := newTestExporter(t, server.URL) + if err := exporter.Refresh(context.Background()); err != nil { + t.Fatalf("first refresh failed: %v", err) + } + count.Store(3) + if err := exporter.Refresh(context.Background()); err != nil { + t.Fatalf("reset refresh failed: %v", err) + } + metrics := collectText(t, exporter) + expected := `hugegraph_http_requests_total{method="GET",result="total",route="/graphspaces/DEFAULT/graphs/graph/vertices"} 3` + if !strings.Contains(metrics, expected) { + t.Fatalf("counter reset was not preserved: %s", metrics) + } +} + +func TestRefreshRejectsOversizedAndSlowResponses(t *testing.T) { + t.Run("body limit", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(bytes.Repeat([]byte("x"), 129)) + })) + t.Cleanup(server.Close) + + config := testConfig(server.URL) + config.MaxBodyBytes = 128 + exporter, err := New(config, log.New(&bytes.Buffer{}, "", 0)) + if err != nil { + t.Fatalf("new exporter: %v", err) + } + if err := exporter.Refresh(context.Background()); err == nil { + t.Fatal("expected body-size error") + } + if metrics := collectText(t, exporter); !strings.Contains(metrics, `stage="body_size"} 1`) { + t.Fatalf("body-size error not classified: %s", metrics) + } + }) + + t.Run("timeout", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + time.Sleep(100 * time.Millisecond) + _, _ = w.Write([]byte(fullFixture(1))) + })) + t.Cleanup(server.Close) + + config := testConfig(server.URL) + config.RequestTimeout = 10 * time.Millisecond + exporter, err := New(config, log.New(&bytes.Buffer{}, "", 0)) + if err != nil { + t.Fatalf("new exporter: %v", err) + } + if err := exporter.Refresh(context.Background()); err == nil { + t.Fatal("expected timeout") + } + if metrics := collectText(t, exporter); !strings.Contains(metrics, `stage="timeout"} 1`) { + t.Fatalf("timeout not classified: %s", metrics) + } + }) +} + +func TestCardinalityFailurePreservesLastSnapshot(t *testing.T) { + var payload atomic.Value + payload.Store(fullFixture(5)) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(payload.Load().(string))) + })) + t.Cleanup(server.Close) + + config := testConfig(server.URL) + config.MaxRoutes = 1 + exporter, err := New(config, log.New(&bytes.Buffer{}, "", 0)) + if err != nil { + t.Fatalf("new exporter: %v", err) + } + if err := exporter.Refresh(context.Background()); err != nil { + t.Fatalf("initial refresh failed: %v", err) + } + payload.Store(twoRouteFixture()) + if err := exporter.Refresh(context.Background()); err == nil { + t.Fatal("expected cardinality error") + } + metrics := collectText(t, exporter) + for _, expected := range []string{ + `stage="cardinality"} 1`, + `route="/graphspaces/DEFAULT/graphs/graph/vertices"} 5`, + } { + if !strings.Contains(metrics, expected) { + t.Fatalf("missing %q after cardinality failure: %s", expected, metrics) + } + } +} + +func newTestExporter(t *testing.T, endpoint string) *Exporter { + t.Helper() + return newTestExporterWithLogger(t, endpoint, log.New(&bytes.Buffer{}, "", 0)) +} + +func newTestExporterWithLogger(t *testing.T, endpoint string, logger *log.Logger) *Exporter { + t.Helper() + exporter, err := New(testConfig(endpoint), logger) + if err != nil { + t.Fatalf("new exporter: %v", err) + } + return exporter +} + +func testConfig(endpoint string) Config { + config := DefaultConfig() + config.Endpoint = endpoint + config.Username = testUsername + config.Password = testPassword + config.Now = func() time.Time { return time.Unix(1700000000, 0) } + return config +} + +func collectText(t *testing.T, exporter *Exporter) string { + t.Helper() + output, err := testutil.CollectAndFormat(exporter, expfmt.TypeTextPlain, + "hugegraph_collection_up", + "hugegraph_collection_stale", + "hugegraph_collection_last_success_unixtime_seconds", + "hugegraph_collection_errors_total", + "hugegraph_http_requests_total", + "hugegraph_http_response_time_ms", + "hugegraph_transaction_commits_total", + "hugegraph_gremlin_errors_total", + "hugegraph_tasks_pending", + "hugegraph_task_workers", + "hugegraph_cache_events_total", + "hugegraph_cache_entries", + ) + if err != nil { + t.Fatalf("collect metrics: %v", err) + } + return string(output) +} + +func fullFixture(requestCount int64) string { + return fmt.Sprintf(`{ + "gauges": { + "org.apache.hugegraph.backend.cache.Cache.vertices-DEFAULT/hugegraph.hits": {"value": 7}, + "org.apache.hugegraph.backend.cache.Cache.vertices-DEFAULT/hugegraph.capacity": {"value": 100}, + "org.apache.hugegraph.task.TaskManager.pending-tasks": {"value": 3}, + "org.apache.hugegraph.task.TaskManager.workers": {"value": 4} + }, + "counters": { + "graphspaces/DEFAULT/graphs/graph/vertices/GET/TOTAL_COUNTER": {"count": %d}, + "metrics/GET/TOTAL_COUNTER": {"count": 99} + }, + "histograms": { + "graphspaces/DEFAULT/graphs/graph/vertices/GET/RESPONSE_TIME_HISTOGRAM": { + "count": %d, + "snapshot": {"max": 30, "mean": 12.5, "95thPercentile": 20, "99thPercentile": 28} + }, + "metrics/GET/RESPONSE_TIME_HISTOGRAM": { + "count": 1, + "snapshot": {"max": 2, "mean": 2, "95thPercentile": 2, "99thPercentile": 2} + } + }, + "meters": { + "org.apache.hugegraph.api.API.commit-succeed": {"count": 9}, + "org.apache.tinkerpop.gremlin.server.GremlinServer.errors": {"count": 2} + }, + "timers": { + "org.apache.hugegraph.example.ignored-timer": {"count": 1, "snapshot": {"max": 1}} + } +}`, requestCount, requestCount) +} + +func twoRouteFixture() string { + return `{ + "gauges": {}, + "counters": { + "graphspaces/DEFAULT/graphs/graph/vertices/GET/TOTAL_COUNTER": {"count": 6}, + "graphspaces/DEFAULT/graphs/graph/edges/GET/TOTAL_COUNTER": {"count": 2} + }, + "histograms": {}, + "meters": {}, + "timers": {} +}` +} diff --git a/addons/hugegraph/exporter/go.mod b/addons/hugegraph/exporter/go.mod new file mode 100644 index 0000000000..0b07d3c397 --- /dev/null +++ b/addons/hugegraph/exporter/go.mod @@ -0,0 +1,20 @@ +module github.com/apecloud/kubeblocks-addons/addons/hugegraph/exporter + +go 1.22 + +require ( + github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/common v0.55.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + golang.org/x/sys v0.22.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect +) diff --git a/addons/hugegraph/exporter/go.sum b/addons/hugegraph/exporter/go.sum new file mode 100644 index 0000000000..d5318cf863 --- /dev/null +++ b/addons/hugegraph/exporter/go.sum @@ -0,0 +1,24 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/addons/hugegraph/exporter/parser.go b/addons/hugegraph/exporter/parser.go new file mode 100644 index 0000000000..03a40c192b --- /dev/null +++ b/addons/hugegraph/exporter/parser.go @@ -0,0 +1,389 @@ +package exporter + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "math" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + cacheMetricPrefix = "org.apache.hugegraph.backend.cache.Cache." + taskMetricPrefix = "org.apache.hugegraph.task.TaskManager." +) + +var ( + allowedMethods = map[string]struct{}{ + "DELETE": {}, "GET": {}, "HEAD": {}, "OPTIONS": {}, + "PATCH": {}, "POST": {}, "PUT": {}, + } + boundedLabelPattern = regexp.MustCompile(`^[A-Za-z0-9_./:{}-]{1,160}$`) + transactionResults = map[string]string{ + "commit-succeed": "succeed", + "illegal-arg": "illegal_arg", + "expected-error": "expected_error", + "unknown-error": "unknown_error", + } +) + +type payload struct { + Gauges map[string]json.RawMessage + Counters map[string]json.RawMessage + Histograms map[string]json.RawMessage + Meters map[string]json.RawMessage + Timers map[string]json.RawMessage +} + +type parseError struct { + stage string +} + +func parsePayload(body []byte, maxRoutes, maxCaches int) ([]metricSample, *parseError) { + var envelope map[string]json.RawMessage + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.UseNumber() + if err := decoder.Decode(&envelope); err != nil { + return nil, &parseError{stage: "decode"} + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + return nil, &parseError{stage: "decode"} + } + + required := []string{"gauges", "counters", "histograms", "meters", "timers"} + for _, key := range required { + if _, ok := envelope[key]; !ok { + return nil, &parseError{stage: "schema"} + } + } + + parsed := payload{} + if err := decodeMetricMap(envelope["gauges"], &parsed.Gauges); err != nil { + return nil, &parseError{stage: "schema"} + } + if err := decodeMetricMap(envelope["counters"], &parsed.Counters); err != nil { + return nil, &parseError{stage: "schema"} + } + if err := decodeMetricMap(envelope["histograms"], &parsed.Histograms); err != nil { + return nil, &parseError{stage: "schema"} + } + if err := decodeMetricMap(envelope["meters"], &parsed.Meters); err != nil { + return nil, &parseError{stage: "schema"} + } + if err := decodeMetricMap(envelope["timers"], &parsed.Timers); err != nil { + return nil, &parseError{stage: "schema"} + } + + return parsed.samples(maxRoutes, maxCaches) +} + +func decodeMetricMap(raw json.RawMessage, target *map[string]json.RawMessage) error { + if err := json.Unmarshal(raw, target); err != nil || *target == nil { + return fmt.Errorf("invalid metric map") + } + return nil +} + +func (payload payload) samples(maxRoutes, maxCaches int) ([]metricSample, *parseError) { + var samples []metricSample + routes := make(map[string]struct{}) + seenSeries := make(map[string]struct{}) + + for _, key := range sortedKeys(payload.Counters) { + route, method, result, matched := parseHTTPKey(key, map[string]string{ + "TOTAL_COUNTER": "total", + "SUCCESS_COUNTER": "success", + "FAILED_COUNTER": "failed", + }) + if !matched || isMetricsRoute(route) { + continue + } + if !boundedLabel(route) { + return nil, &parseError{stage: "cardinality"} + } + routes[route] = struct{}{} + if len(routes) > maxRoutes { + return nil, &parseError{stage: "cardinality"} + } + value, ok := metricNumber(payload.Counters[key], "count", "value") + if !ok || value < 0 { + return nil, &parseError{stage: "schema"} + } + seriesKey := "http-counter\x00" + route + "\x00" + method + "\x00" + result + if !addSeries(seenSeries, seriesKey) { + return nil, &parseError{stage: "schema"} + } + samples = append(samples, metricSample{ + desc: httpRequestsDesc, valueType: prometheus.CounterValue, + value: value, labels: []string{route, method, result}, + }) + } + + for _, key := range sortedKeys(payload.Histograms) { + route, method, _, matched := parseHTTPKey(key, map[string]string{ + "RESPONSE_TIME_HISTOGRAM": "latency", + }) + if !matched || isMetricsRoute(route) { + continue + } + if !boundedLabel(route) { + return nil, &parseError{stage: "cardinality"} + } + routes[route] = struct{}{} + if len(routes) > maxRoutes { + return nil, &parseError{stage: "cardinality"} + } + stats := []struct { + name string + aliases []string + }{ + {name: "mean", aliases: []string{"mean"}}, + {name: "p95", aliases: []string{"p95", "95thPercentile"}}, + {name: "p99", aliases: []string{"p99", "99thPercentile"}}, + {name: "max", aliases: []string{"max"}}, + } + for _, stat := range stats { + value, ok := metricNumber(payload.Histograms[key], stat.aliases...) + if !ok || value < 0 { + return nil, &parseError{stage: "schema"} + } + seriesKey := "http-latency\x00" + route + "\x00" + method + "\x00" + stat.name + if !addSeries(seenSeries, seriesKey) { + return nil, &parseError{stage: "schema"} + } + samples = append(samples, metricSample{ + desc: httpResponseTimeDesc, valueType: prometheus.GaugeValue, + value: value, labels: []string{route, method, stat.name}, + }) + } + } + + samples, err := payload.appendMeters(samples, seenSeries) + if err != nil { + return nil, err + } + samples, err = payload.appendGauges(samples, seenSeries, maxCaches) + if err != nil { + return nil, err + } + return samples, nil +} + +func (payload payload) appendMeters(samples []metricSample, seenSeries map[string]struct{}) ([]metricSample, *parseError) { + var gremlinErrors float64 + var hasGremlinErrors bool + for _, key := range sortedKeys(payload.Meters) { + value, ok := metricNumber(payload.Meters[key], "count", "value") + if !ok || value < 0 { + continue + } + for suffix, result := range transactionResults { + if strings.HasSuffix(key, "."+suffix) { + seriesKey := "transaction\x00" + result + if !addSeries(seenSeries, seriesKey) { + return nil, &parseError{stage: "schema"} + } + samples = append(samples, metricSample{ + desc: transactionCommitsDesc, valueType: prometheus.CounterValue, + value: value, labels: []string{result}, + }) + } + } + lowerKey := strings.ToLower(key) + if strings.Contains(lowerKey, "gremlin") && strings.HasSuffix(lowerKey, ".errors") { + gremlinErrors += value + hasGremlinErrors = true + } + } + if hasGremlinErrors { + samples = append(samples, metricSample{ + desc: gremlinErrorsDesc, valueType: prometheus.CounterValue, + value: gremlinErrors, + }) + } + return samples, nil +} + +func (payload payload) appendGauges(samples []metricSample, seenSeries map[string]struct{}, maxCaches int) ([]metricSample, *parseError) { + caches := make(map[string]struct{}) + for _, key := range sortedKeys(payload.Gauges) { + value, ok := metricNumber(payload.Gauges[key], "value", "count") + if !ok || value < 0 { + continue + } + switch key { + case taskMetricPrefix + "pending-tasks": + samples = append(samples, metricSample{ + desc: tasksPendingDesc, valueType: prometheus.GaugeValue, + value: value, labels: []string{"task"}, + }) + case taskMetricPrefix + "workers": + samples = append(samples, metricSample{ + desc: taskWorkersDesc, valueType: prometheus.GaugeValue, + value: value, labels: []string{"task", "configured"}, + }) + } + + if !strings.HasPrefix(key, cacheMetricPrefix) { + continue + } + identity, rawState, ok := splitCacheMetric(strings.TrimPrefix(key, cacheMetricPrefix)) + if !ok { + continue + } + cache, graph, ok := splitCacheIdentity(identity) + if !ok || !boundedLabel(cache) || !boundedLabel(graph) { + return nil, &parseError{stage: "cardinality"} + } + cacheKey := graph + "\x00" + cache + caches[cacheKey] = struct{}{} + if len(caches) > maxCaches { + return nil, &parseError{stage: "cardinality"} + } + + switch rawState { + case "hits", "miss", "expire": + event := strings.TrimSuffix(rawState, "s") + seriesKey := "cache-event\x00" + cacheKey + "\x00" + event + if !addSeries(seenSeries, seriesKey) { + return nil, &parseError{stage: "schema"} + } + samples = append(samples, metricSample{ + desc: cacheEventsDesc, valueType: prometheus.CounterValue, + value: value, labels: []string{graph, cache, event}, + }) + case "size", "capacity": + seriesKey := "cache-entry\x00" + cacheKey + "\x00" + rawState + if !addSeries(seenSeries, seriesKey) { + return nil, &parseError{stage: "schema"} + } + samples = append(samples, metricSample{ + desc: cacheEntriesDesc, valueType: prometheus.GaugeValue, + value: value, labels: []string{graph, cache, rawState}, + }) + } + } + return samples, nil +} + +func parseHTTPKey(key string, suffixes map[string]string) (string, string, string, bool) { + parts := strings.Split(key, "/") + if len(parts) < 3 { + return "", "", "", false + } + result, ok := suffixes[parts[len(parts)-1]] + if !ok { + return "", "", "", false + } + method := strings.ToUpper(parts[len(parts)-2]) + if _, ok := allowedMethods[method]; !ok { + return "", "", "", false + } + route := "/" + strings.TrimPrefix(strings.Join(parts[:len(parts)-2], "/"), "/") + return route, method, result, true +} + +func isMetricsRoute(route string) bool { + return route == "/metrics" || strings.HasPrefix(route, "/metrics/") +} + +func splitCacheMetric(metric string) (string, string, bool) { + index := strings.LastIndexByte(metric, '.') + if index <= 0 || index == len(metric)-1 { + return "", "", false + } + return metric[:index], metric[index+1:], true +} + +func splitCacheIdentity(identity string) (string, string, bool) { + index := strings.IndexByte(identity, '-') + if index <= 0 || index == len(identity)-1 { + return "", "", false + } + return identity[:index], identity[index+1:], true +} + +func metricNumber(raw json.RawMessage, fields ...string) (float64, bool) { + var value any + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + if err := decoder.Decode(&value); err != nil { + return 0, false + } + if number, ok := numericValue(value); ok { + return number, true + } + object, ok := value.(map[string]any) + if !ok { + return 0, false + } + if number, ok := numberFromMap(object, fields...); ok { + return number, true + } + for _, nestedKey := range []string{"snapshot", "value"} { + if nested, ok := object[nestedKey].(map[string]any); ok { + if number, ok := numberFromMap(nested, fields...); ok { + return number, true + } + } + } + return 0, false +} + +func numberFromMap(object map[string]any, fields ...string) (float64, bool) { + for _, field := range fields { + for key, value := range object { + if strings.EqualFold(key, field) { + return numericValue(value) + } + } + } + return 0, false +} + +func numericValue(value any) (float64, bool) { + var number float64 + switch typed := value.(type) { + case json.Number: + parsed, err := strconv.ParseFloat(string(typed), 64) + if err != nil { + return 0, false + } + number = parsed + case float64: + number = typed + case int: + number = float64(typed) + default: + return 0, false + } + return number, !math.IsNaN(number) && !math.IsInf(number, 0) +} + +func boundedLabel(value string) bool { + return boundedLabelPattern.MatchString(value) +} + +func addSeries(seen map[string]struct{}, key string) bool { + if _, ok := seen[key]; ok { + return false + } + seen[key] = struct{}{} + return true +} + +func sortedKeys[V any](values map[string]V) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} diff --git a/addons/hugegraph/templates/_helpers.tpl b/addons/hugegraph/templates/_helpers.tpl index 988cf1d32b..05edbf8556 100644 --- a/addons/hugegraph/templates/_helpers.tpl +++ b/addons/hugegraph/templates/_helpers.tpl @@ -51,3 +51,7 @@ hugegraph-backup-policy-template {{- define "hugegraph.image" -}} {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} {{- end }} + +{{- define "hugegraph.exporterImage" -}} +{{ .Values.exporter.image.registry | default "docker.io" }}/{{ .Values.exporter.image.repository }}:{{ .Values.exporter.image.tag }} +{{- end }} diff --git a/addons/hugegraph/templates/cmpd.yaml b/addons/hugegraph/templates/cmpd.yaml index d390d213ef..4f6f651d83 100644 --- a/addons/hugegraph/templates/cmpd.yaml +++ b/addons/hugegraph/templates/cmpd.yaml @@ -12,6 +12,11 @@ spec: description: Apache HugeGraph 1.7.0 standalone server with RocksDB. serviceKind: hugegraph serviceVersion: {{ .Values.serviceVersion }} + exporter: + containerName: hugegraph-exporter + scrapePath: /metrics + scrapePort: metrics + scrapeScheme: http minReadySeconds: 10 replicasLimit: minReplicas: 1 @@ -125,3 +130,30 @@ spec: mountPath: {{ .Values.dataMountPath }} - name: scripts mountPath: /scripts + - name: hugegraph-exporter + image: {{ include "hugegraph.exporterImage" . }} + imagePullPolicy: {{ .Values.exporter.image.pullPolicy }} + env: + - name: HUGEGRAPH_USERNAME + value: $(ADMIN_USER) + - name: HUGEGRAPH_PASSWORD + value: $(ADMIN_PASSWORD) + ports: + - name: metrics + containerPort: 9404 + readinessProbe: + httpGet: + path: /healthz + port: metrics + failureThreshold: 3 + periodSeconds: 10 + timeoutSeconds: 3 + livenessProbe: + httpGet: + path: /healthz + port: metrics + failureThreshold: 6 + periodSeconds: 15 + timeoutSeconds: 3 + resources: + {{- toYaml .Values.exporter.resources | nindent 10 }} diff --git a/addons/hugegraph/templates/monitordefinition.yaml b/addons/hugegraph/templates/monitordefinition.yaml new file mode 100644 index 0000000000..d540773ff0 --- /dev/null +++ b/addons/hugegraph/templates/monitordefinition.yaml @@ -0,0 +1,21 @@ +apiVersion: monitor.kubeblocks.io/v1alpha1 +kind: MonitorDefinition +metadata: + name: hugegraph + labels: + {{- include "hugegraph.labels" . | nindent 4 }} +spec: + clusterDefRef: hugegraph + components: + - componentDefRef: hugegraph + regexEnabled: true + componentKinds: + - hugegraph.* + collectors: + - name: hugegraph-exporter + monitorType: prometheus + metrics: + collectionInterval: 30s + prometheus: + scrapePort: metrics + metricsPath: /metrics diff --git a/addons/hugegraph/tests/contract_test.sh b/addons/hugegraph/tests/contract_test.sh index e83bd693c7..b0cf6a2fe7 100644 --- a/addons/hugegraph/tests/contract_test.sh +++ b/addons/hugegraph/tests/contract_test.sh @@ -11,6 +11,13 @@ fail() { exit 1 } +assert_equal() { + local actual=$1 + local expected=$2 + local message=$3 + [[ "${actual}" == "${expected}" ]] || fail "${message}: got ${actual}, want ${expected}" +} + assert_file() { [[ -f "$1" ]] || fail "missing file: $1" } @@ -35,6 +42,7 @@ required_files=( "${ADDON_DIR}/Chart.yaml" "${ADDON_DIR}/values.yaml" "${ADDON_DIR}/templates/cmpd.yaml" + "${ADDON_DIR}/templates/monitordefinition.yaml" "${ADDON_DIR}/templates/cmpv.yaml" "${ADDON_DIR}/templates/clusterdefinition.yaml" "${ADDON_DIR}/templates/backuppolicytemplate.yaml" @@ -45,6 +53,8 @@ required_files=( "${ADDON_DIR}/scripts/backup.sh" "${ADDON_DIR}/scripts/restore.sh" "${ADDON_DIR}/tests/scripts_test.sh" + "${ADDON_DIR}/exporter/Dockerfile" + "${ADDON_DIR}/exporter/go.mod" "${CLUSTER_DIR}/Chart.yaml" "${CLUSTER_DIR}/templates/cluster.yaml" "${ROOT_DIR}/examples/hugegraph/cluster.yaml" @@ -81,6 +91,10 @@ assert_contains "${ADDON_DIR}/scripts/shutdown.sh" '\.kb-prestop\.log' assert_contains "${ADDON_DIR}/templates/cmpd.yaml" 'terminationGracePeriodSeconds: 30' assert_contains "${ADDON_DIR}/templates/cmpd.yaml" 'preStop:' assert_contains "${ADDON_DIR}/templates/cmpd.yaml" '/scripts/shutdown\.sh' +assert_contains "${ADDON_DIR}/templates/cmpd.yaml" 'name: hugegraph-exporter' +assert_contains "${ADDON_DIR}/templates/cmpd.yaml" 'containerName: hugegraph-exporter' +assert_contains "${ADDON_DIR}/templates/cmpd.yaml" 'scrapePort: metrics' +assert_contains "${ADDON_DIR}/templates/cmpd.yaml" 'scrapePath: /metrics' assert_not_contains_tree 'volume[-_ ]snapshot' assert_not_contains_tree 'snapshotVolumes:[[:space:]]*true' @@ -97,7 +111,7 @@ trap 'rm -f "${definition_render}" "${cluster_render}"' EXIT helm template hugegraph-def "${ADDON_DIR}" --namespace kb-system >"${definition_render}" helm template hugegraph "${CLUSTER_DIR}" --namespace demo >"${cluster_render}" -for kind in ClusterDefinition ComponentDefinition ComponentVersion BackupPolicyTemplate ActionSet; do +for kind in ClusterDefinition ComponentDefinition ComponentVersion BackupPolicyTemplate ActionSet MonitorDefinition; do rg -q "^kind: ${kind}$" "${definition_render}" || fail "render misses ${kind}" done @@ -106,6 +120,56 @@ assert_contains "${definition_render}" 'image: docker.io/hugegraph/hugegraph:1.7 assert_contains "${definition_render}" 'mountPath: /hugegraph-data' assert_contains "${definition_render}" 'containerPort: 8080' assert_contains "${definition_render}" 'containerPort: 8182' +assert_contains "${definition_render}" 'name: hugegraph-exporter' +assert_contains "${definition_render}" 'image: docker.io/apecloud/hugegraph-exporter:0.1.1' +assert_contains "${definition_render}" 'containerPort: 9404' +assert_contains "${definition_render}" 'name: HUGEGRAPH_USERNAME' +assert_contains "${definition_render}" 'value: \$\(ADMIN_USER\)' +assert_contains "${definition_render}" 'name: HUGEGRAPH_PASSWORD' +assert_contains "${definition_render}" 'value: \$\(ADMIN_PASSWORD\)' +assert_contains "${definition_render}" 'cpu: 20m' +assert_contains "${definition_render}" 'memory: 32Mi' +assert_contains "${definition_render}" 'cpu: 100m' +assert_contains "${definition_render}" 'memory: 128Mi' +assert_contains "${definition_render}" 'kind: MonitorDefinition' +assert_contains "${definition_render}" 'clusterDefRef: hugegraph' +assert_contains "${definition_render}" 'componentDefRef: hugegraph' +assert_contains "${definition_render}" 'scrapePort: metrics' +assert_contains "${definition_render}" 'metricsPath: /metrics' +assert_contains "${definition_render}" 'collectionInterval: 30s' + +assert_equal \ + "$(yq 'select(.kind == "ComponentDefinition") | .spec.runtime.containers | length' "${definition_render}")" \ + "2" \ + "component container count" +assert_equal \ + "$(yq 'select(.kind == "ComponentDefinition") | .spec.volumes | length' "${definition_render}")" \ + "1" \ + "component volume count" +assert_equal \ + "$(yq 'select(.kind == "ComponentDefinition") | .spec.volumes[0].name' "${definition_render}")" \ + "data" \ + "component data volume" +assert_equal \ + "$(yq 'select(.kind == "ComponentDefinition") | .spec.runtime.containers[] | select(.name == "hugegraph") | .volumeMounts | length' "${definition_render}")" \ + "2" \ + "main container volume mounts" +assert_equal \ + "$(yq 'select(.kind == "ComponentDefinition") | .spec.runtime.containers[] | select(.name == "hugegraph-exporter") | (.volumeMounts // []) | length' "${definition_render}")" \ + "0" \ + "exporter PVC mounts" +assert_equal \ + "$(yq 'select(.kind == "ComponentDefinition") | .spec.runtime.containers[] | select(.name == "hugegraph-exporter") | .env | length' "${definition_render}")" \ + "2" \ + "exporter credential env count" +assert_equal \ + "$(yq 'select(.kind == "ComponentDefinition") | .spec.exporter.containerName' "${definition_render}")" \ + "hugegraph-exporter" \ + "spec.exporter container" +assert_equal \ + "$(yq 'select(.kind == "MonitorDefinition") | .spec.components[0].collectors[0].name' "${definition_render}")" \ + "hugegraph-exporter" \ + "MonitorDefinition collector" assert_contains "${definition_render}" 'terminationGracePeriodSeconds: 30' assert_contains "${definition_render}" 'preStop:' assert_contains "${definition_render}" '/scripts/shutdown\.sh' @@ -113,6 +177,18 @@ assert_contains "${cluster_render}" 'clusterDef: hugegraph' assert_contains "${cluster_render}" 'topology: standalone' assert_contains "${cluster_render}" 'replicas: 1' assert_contains "${cluster_render}" 'serviceVersion: [\"]?1\.7\.0[\"]?' +assert_contains "${ADDON_DIR}/.helmignore" '^exporter/' + +package_dir=$(mktemp -d) +trap 'rm -f "${definition_render}" "${cluster_render}"; rm -rf "${package_dir}"' EXIT +helm package "${ADDON_DIR}" -d "${package_dir}" >/dev/null +package_tgz=$(echo "${package_dir}"/hugegraph-*.tgz) +[[ -f "${package_tgz}" ]] || fail "helm package missing" +if tar tzf "${package_tgz}" | grep -E '(^|/)exporter/'; then + fail "packaged chart includes exporter/ source" +fi +package_bytes=$(wc -c < "${package_tgz}" | tr -d ' ') +[[ "${package_bytes}" -lt 100000 ]] || fail "packaged chart too large: ${package_bytes} bytes" "${ADDON_DIR}/tests/scripts_test.sh" diff --git a/addons/hugegraph/values.yaml b/addons/hugegraph/values.yaml index 964367a48c..de9e24c0e6 100644 --- a/addons/hugegraph/values.yaml +++ b/addons/hugegraph/values.yaml @@ -7,6 +7,20 @@ image: tag: "1.7.0" pullPolicy: IfNotPresent +exporter: + image: + registry: docker.io + repository: apecloud/hugegraph-exporter + tag: "0.1.1" + pullPolicy: IfNotPresent + resources: + requests: + cpu: 20m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi + serviceVersion: "1.7.0" dataMountPath: /hugegraph-data From 4069790f76f788f31d9635f293350272b258249e Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Wed, 19 Aug 2026 01:52:24 +0800 Subject: [PATCH 05/16] feat(hugegraph): add distributed PD/Store/Server topology (#3414) Co-authored-by: Amber --- .../hugegraph/templates/cluster.yaml | 19 +++- addons-cluster/hugegraph/values.yaml | 6 ++ addons/hugegraph/DESIGN-distributed.zh.md | 62 +++++++++++ addons/hugegraph/scripts/start-pd.sh | 47 ++++++++ addons/hugegraph/scripts/start-server.sh | 29 +++++ addons/hugegraph/scripts/start-store.sh | 52 +++++++++ addons/hugegraph/templates/_helpers.tpl | 38 ++++++- .../templates/clusterdefinition.yaml | 18 ++++ addons/hugegraph/templates/cmpd-pd.yaml | 100 ++++++++++++++++++ addons/hugegraph/templates/cmpd-server.yaml | 83 +++++++++++++++ addons/hugegraph/templates/cmpd-store.yaml | 100 ++++++++++++++++++ addons/hugegraph/templates/cmpv.yaml | 24 +++++ .../hugegraph/templates/script-template.yaml | 6 ++ addons/hugegraph/tests/contract_test.sh | 58 ++++++++-- addons/hugegraph/values.yaml | 13 +++ examples/hugegraph/README.md | 3 +- examples/hugegraph/cluster-distributed.yaml | 58 ++++++++++ 17 files changed, 703 insertions(+), 13 deletions(-) create mode 100644 addons/hugegraph/DESIGN-distributed.zh.md create mode 100644 addons/hugegraph/scripts/start-pd.sh create mode 100644 addons/hugegraph/scripts/start-server.sh create mode 100644 addons/hugegraph/scripts/start-store.sh create mode 100644 addons/hugegraph/templates/cmpd-pd.yaml create mode 100644 addons/hugegraph/templates/cmpd-server.yaml create mode 100644 addons/hugegraph/templates/cmpd-store.yaml create mode 100644 examples/hugegraph/cluster-distributed.yaml diff --git a/addons-cluster/hugegraph/templates/cluster.yaml b/addons-cluster/hugegraph/templates/cluster.yaml index ee27e18518..2bfc633d75 100644 --- a/addons-cluster/hugegraph/templates/cluster.yaml +++ b/addons-cluster/hugegraph/templates/cluster.yaml @@ -6,11 +6,28 @@ metadata: labels: {{ include "kblib.clusterLabels" . | nindent 4 }} spec: clusterDef: hugegraph - topology: standalone + topology: {{ .Values.topology | default "standalone" }} terminationPolicy: {{ .Values.extra.terminationPolicy }} componentSpecs: + {{- if eq (.Values.topology | default "standalone") "distributed" }} + - name: pd + serviceVersion: {{ .Values.version | quote }} + replicas: {{ .Values.distributed.pdReplicas | default 3 }} + {{- include "kblib.componentResources" . | nindent 6 }} + {{- include "kblib.componentStorages" . | nindent 6 }} + - name: store + serviceVersion: {{ .Values.version | quote }} + replicas: {{ .Values.distributed.storeReplicas | default 3 }} + {{- include "kblib.componentResources" . | nindent 6 }} + {{- include "kblib.componentStorages" . | nindent 6 }} + - name: server + serviceVersion: {{ .Values.version | quote }} + replicas: {{ .Values.distributed.serverReplicas | default 1 }} + {{- include "kblib.componentResources" . | nindent 6 }} + {{- else }} - name: server serviceVersion: {{ .Values.version | quote }} replicas: 1 {{- include "kblib.componentResources" . | nindent 6 }} {{- include "kblib.componentStorages" . | nindent 6 }} + {{- end }} diff --git a/addons-cluster/hugegraph/values.yaml b/addons-cluster/hugegraph/values.yaml index b8f5ca9fc6..2f6d7a2272 100644 --- a/addons-cluster/hugegraph/values.yaml +++ b/addons-cluster/hugegraph/values.yaml @@ -1,6 +1,12 @@ version: "1.7.0" +topology: standalone replicas: 1 +distributed: + pdReplicas: 3 + storeReplicas: 3 + serverReplicas: 1 + cpu: 1 memory: 2 diff --git a/addons/hugegraph/DESIGN-distributed.zh.md b/addons/hugegraph/DESIGN-distributed.zh.md new file mode 100644 index 0000000000..0069c0c41d --- /dev/null +++ b/addons/hugegraph/DESIGN-distributed.zh.md @@ -0,0 +1,62 @@ +# HugeGraph 1.7.0 分布式 Topology 设计 + +依据 `docs/addon-api/03-cluster-definition.md`、`06-variables-and-services.md`、 +`12a-minimum-acceptance.md`、`12b-claimed-only-acceptance.md`。 + +## 1. 拓扑含义 + +`distributed` 是用户可选的第二种部署形态:PD + Store + Server。 +这不是 sharding。PD / Store 的多副本是同一 Raft 组里的 `replicas`。 + +官方 1.7.0 运行时依赖是 PD healthy -> Store -> Server。 +PD 启动又需要事先知道 Store 的稳定 FQDN 列表,所以对象创建上 PD 和 Store +放在同一 provision 阶段(逗号=并行),Server 后置。Store 启动脚本等 PD +`/v1/health`。这符合合同:阶段内并行、阶段间有序。 + +```text +orders.provision: [pd,store, server] +orders.terminate: [server, store,pd] +orders.update: [pd,store, server] +``` + +`standalone` 仍是唯一 default。未指定 topology 时必须落到 standalone。 + +## 2. 组件与名字 + +| topology component | CmpD 名 | 正则 | 镜像 | +| --- | --- | --- | --- | +| standalone `server` | `hugegraph-{{ ver }}` | `^hugegraph-[0-9]` | `hugegraph/hugegraph:1.7.0` | +| distributed `pd` | `hugegraph-pd-{{ ver }}` | `^hugegraph-pd-` | `hugegraph/pd:1.7.0` | +| distributed `store` | `hugegraph-store-{{ ver }}` | `^hugegraph-store-` | `hugegraph/store:1.7.0` | +| distributed `server` | `hugegraph-server-{{ ver }}` | `^hugegraph-server-` | `hugegraph/server:1.7.0` | + +standalone 原来的 `^hugegraph-` 会误伤 `hugegraph-pd-` / `hugegraph-store-` / +`hugegraph-server-`,必须收窄。这是合同要求:`compDef` 正则要稳定命中唯一集合。 + +## 3. Raft 身份 + +官方用 hostname 当 Raft 成员 ID(`pd0:8610`)。合同要求不要把不可重算的 +Pod 名 / clusterUID 写进引擎成员 ID。这里用 KB `podFQDNs` 生成 +`{podFQDN}:port`,恢复或同名重建时只要 FQDN 规则不变就可以重算。 + +本节点地址从 `podFQDNs` 里按 `hostname` / `metadata.name` 前缀匹配,不手写 +headless Service 名。 + +## 4. 本 PR 声明 / 不声明 + +声明: + +- `distributed` topology 可渲染,orders 闭合 +- 三组件 CmpD / ComponentVersion / example 名字对齐 +- PD / Store / Server 用官方 `HG_*` 环境变量启动 +- 最小规格:pd 3、store 3、server 1 + +不声明(12b,本轮不验收): + +- distributed backup / restore / PITR +- Store scale-in / rebalance +- PD / Store switchover、roleProbe +- TLS、reconfigure +- Hubble + +BackupPolicyTemplate 继续只命中 standalone server CmpD。 diff --git a/addons/hugegraph/scripts/start-pd.sh b/addons/hugegraph/scripts/start-pd.sh new file mode 100644 index 0000000000..2f4d3a4489 --- /dev/null +++ b/addons/hugegraph/scripts/start-pd.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -euo pipefail + +: "${PD_POD_FQDNS:?PD_POD_FQDNS is required}" +: "${STORE_POD_FQDNS:?STORE_POD_FQDNS is required}" + +pod_name=${POD_NAME:-$(hostname)} + +append_port() { + local list=$1 + local port=$2 + local out="" host + IFS=',' read -ra hosts <<< "${list}" + for host in "${hosts[@]}"; do + [[ -n "${host}" ]] || continue + [[ -n "${out}" ]] && out+="," + out+="${host}:${port}" + done + printf '%s' "${out}" +} + +self="" +IFS=',' read -ra hosts <<< "${PD_POD_FQDNS}" +for host in "${hosts[@]}"; do + short=${host%%.*} + if [[ "${short}" == "${pod_name}" || "${host}" == "${pod_name}" ]]; then + self=${host} + break + fi +done +[[ -n "${self}" ]] || { + echo "cannot map pod ${pod_name} to PD_POD_FQDNS=${PD_POD_FQDNS}" >&2 + exit 1 +} + +export HG_PD_GRPC_HOST="${self}" +export HG_PD_GRPC_PORT="${HG_PD_GRPC_PORT:-8686}" +export HG_PD_REST_PORT="${HG_PD_REST_PORT:-8620}" +export HG_PD_RAFT_ADDRESS="${self}:8610" +export HG_PD_RAFT_PEERS_LIST="$(append_port "${PD_POD_FQDNS}" 8610)" +export HG_PD_INITIAL_STORE_LIST="$(append_port "${STORE_POD_FQDNS}" 8500)" +export HG_PD_DATA_PATH="${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data}" +export HG_PD_INITIAL_STORE_COUNT="${HG_PD_INITIAL_STORE_COUNT:-1}" + +mkdir -p "${HG_PD_DATA_PATH}" +cd /hugegraph-pd +exec /usr/bin/dumb-init -- ./docker-entrypoint.sh diff --git a/addons/hugegraph/scripts/start-server.sh b/addons/hugegraph/scripts/start-server.sh new file mode 100644 index 0000000000..b0fb6b00f3 --- /dev/null +++ b/addons/hugegraph/scripts/start-server.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +: "${PD_POD_FQDNS:?PD_POD_FQDNS is required}" +: "${STORE_POD_FQDNS:?STORE_POD_FQDNS is required}" + +append_port() { + local list=$1 + local port=$2 + local out="" host + IFS=',' read -ra hosts <<< "${list}" + for host in "${hosts[@]}"; do + [[ -n "${host}" ]] || continue + [[ -n "${out}" ]] && out+="," + out+="${host}:${port}" + done + printf '%s' "${out}" +} + +first_store=${STORE_POD_FQDNS%%,*} + +export HG_SERVER_BACKEND=hstore +export HG_SERVER_PD_PEERS="$(append_port "${PD_POD_FQDNS}" 8686)" +export HG_SERVER_USE_PD=true +export HG_SERVER_INIT_STORE_ENABLED=false +export STORE_REST="${first_store}:8520" + +cd /hugegraph-server +exec /usr/bin/dumb-init -- ./docker-entrypoint.sh diff --git a/addons/hugegraph/scripts/start-store.sh b/addons/hugegraph/scripts/start-store.sh new file mode 100644 index 0000000000..cad0bf17bf --- /dev/null +++ b/addons/hugegraph/scripts/start-store.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -euo pipefail + +: "${PD_POD_FQDNS:?PD_POD_FQDNS is required}" +: "${STORE_POD_FQDNS:?STORE_POD_FQDNS is required}" + +pod_name=${POD_NAME:-$(hostname)} + +append_port() { + local list=$1 + local port=$2 + local out="" host + IFS=',' read -ra hosts <<< "${list}" + for host in "${hosts[@]}"; do + [[ -n "${host}" ]] || continue + [[ -n "${out}" ]] && out+="," + out+="${host}:${port}" + done + printf '%s' "${out}" +} + +self="" +IFS=',' read -ra hosts <<< "${STORE_POD_FQDNS}" +for host in "${hosts[@]}"; do + short=${host%%.*} + if [[ "${short}" == "${pod_name}" || "${host}" == "${pod_name}" ]]; then + self=${host} + break + fi +done +[[ -n "${self}" ]] || { + echo "cannot map pod ${pod_name} to STORE_POD_FQDNS=${STORE_POD_FQDNS}" >&2 + exit 1 +} + +export HG_STORE_PD_ADDRESS="$(append_port "${PD_POD_FQDNS}" 8686)" +first_pd=${PD_POD_FQDNS%%,*} +for _ in $(seq 1 60); do + if curl -fsS "http://${first_pd}:8620/v1/health" >/dev/null; then + break + fi + sleep 2 +done +export HG_STORE_GRPC_HOST="${self}" +export HG_STORE_GRPC_PORT="${HG_STORE_GRPC_PORT:-8500}" +export HG_STORE_REST_PORT="${HG_STORE_REST_PORT:-8520}" +export HG_STORE_RAFT_ADDRESS="${self}:8510" +export HG_STORE_DATA_PATH="${HG_STORE_DATA_PATH:-/hugegraph-store/storage}" + +mkdir -p "${HG_STORE_DATA_PATH}" +cd /hugegraph-store +exec /usr/bin/dumb-init -- ./docker-entrypoint.sh diff --git a/addons/hugegraph/templates/_helpers.tpl b/addons/hugegraph/templates/_helpers.tpl index 05edbf8556..b8f9620668 100644 --- a/addons/hugegraph/templates/_helpers.tpl +++ b/addons/hugegraph/templates/_helpers.tpl @@ -33,7 +33,43 @@ hugegraph-{{ .Chart.Version }} {{- end }} {{- define "hugegraph.cmpdPattern" -}} -^hugegraph- +^hugegraph-[0-9] +{{- end }} + +{{- define "hugegraph.pdCmpdName" -}} +hugegraph-pd-{{ .Chart.Version }} +{{- end }} + +{{- define "hugegraph.pdCmpdPattern" -}} +^hugegraph-pd- +{{- end }} + +{{- define "hugegraph.storeCmpdName" -}} +hugegraph-store-{{ .Chart.Version }} +{{- end }} + +{{- define "hugegraph.storeCmpdPattern" -}} +^hugegraph-store- +{{- end }} + +{{- define "hugegraph.serverCmpdName" -}} +hugegraph-server-{{ .Chart.Version }} +{{- end }} + +{{- define "hugegraph.serverCmpdPattern" -}} +^hugegraph-server- +{{- end }} + +{{- define "hugegraph.pdImage" -}} +{{ .Values.image.registry | default "docker.io" }}/{{ .Values.distributed.pd.repository }}:{{ .Values.distributed.pd.tag }} +{{- end }} + +{{- define "hugegraph.storeImage" -}} +{{ .Values.image.registry | default "docker.io" }}/{{ .Values.distributed.store.repository }}:{{ .Values.distributed.store.tag }} +{{- end }} + +{{- define "hugegraph.serverImage" -}} +{{ .Values.image.registry | default "docker.io" }}/{{ .Values.distributed.server.repository }}:{{ .Values.distributed.server.tag }} {{- end }} {{- define "hugegraph.scriptsTemplateName" -}} diff --git a/addons/hugegraph/templates/clusterdefinition.yaml b/addons/hugegraph/templates/clusterdefinition.yaml index 9e7bd5a2dd..7514a324ad 100644 --- a/addons/hugegraph/templates/clusterdefinition.yaml +++ b/addons/hugegraph/templates/clusterdefinition.yaml @@ -13,3 +13,21 @@ spec: components: - name: server compDef: {{ include "hugegraph.cmpdPattern" . }} + - name: distributed + components: + - name: pd + compDef: {{ include "hugegraph.pdCmpdPattern" . }} + - name: store + compDef: {{ include "hugegraph.storeCmpdPattern" . }} + - name: server + compDef: {{ include "hugegraph.serverCmpdPattern" . }} + orders: + provision: + - pd,store + - server + terminate: + - server + - store,pd + update: + - pd,store + - server diff --git a/addons/hugegraph/templates/cmpd-pd.yaml b/addons/hugegraph/templates/cmpd-pd.yaml new file mode 100644 index 0000000000..2ced438ad7 --- /dev/null +++ b/addons/hugegraph/templates/cmpd-pd.yaml @@ -0,0 +1,100 @@ +apiVersion: apps.kubeblocks.io/v1 +kind: ComponentDefinition +metadata: + name: {{ include "hugegraph.pdCmpdName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + annotations: + apps.kubeblocks.io/skip-immutable-check: "true" + {{- include "hugegraph.annotations" . | nindent 4 }} +spec: + provider: kubeblocks + description: Apache HugeGraph 1.7.0 Placement Driver. + serviceKind: hugegraph-pd + serviceVersion: {{ .Values.serviceVersion }} + podManagementPolicy: Parallel + minReadySeconds: 10 + replicasLimit: + minReplicas: 1 + maxReplicas: 5 + services: + - name: default + spec: + ports: + - name: rest + port: 8620 + targetPort: rest + - name: grpc + port: 8686 + targetPort: grpc + - name: raft + port: 8610 + targetPort: raft + scripts: + - name: hugegraph-scripts + template: {{ include "hugegraph.scriptsTemplateName" . }} + namespace: {{ .Release.Namespace }} + volumeName: scripts + defaultMode: 0555 + volumes: + - name: data + vars: + - name: PD_POD_FQDNS + valueFrom: + componentVarRef: + compDef: {{ include "hugegraph.pdCmpdPattern" . }} + optional: false + podFQDNs: Required + - name: STORE_POD_FQDNS + valueFrom: + componentVarRef: + compDef: {{ include "hugegraph.storeCmpdPattern" . }} + optional: false + podFQDNs: Required + runtime: + containers: + - name: pd + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /bin/bash + - /scripts/start-pd.sh + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: HG_PD_DATA_PATH + value: {{ .Values.distributed.pd.dataMountPath }} + ports: + - name: rest + containerPort: 8620 + - name: grpc + containerPort: 8686 + - name: raft + containerPort: 8610 + startupProbe: + httpGet: + path: /v1/health + port: rest + failureThreshold: 36 + periodSeconds: 5 + timeoutSeconds: 3 + readinessProbe: + httpGet: + path: /v1/health + port: rest + failureThreshold: 3 + periodSeconds: 10 + timeoutSeconds: 3 + livenessProbe: + httpGet: + path: /v1/health + port: rest + failureThreshold: 6 + periodSeconds: 15 + timeoutSeconds: 3 + volumeMounts: + - name: data + mountPath: {{ .Values.distributed.pd.dataMountPath }} + - name: scripts + mountPath: /scripts diff --git a/addons/hugegraph/templates/cmpd-server.yaml b/addons/hugegraph/templates/cmpd-server.yaml new file mode 100644 index 0000000000..150ce25ec8 --- /dev/null +++ b/addons/hugegraph/templates/cmpd-server.yaml @@ -0,0 +1,83 @@ +apiVersion: apps.kubeblocks.io/v1 +kind: ComponentDefinition +metadata: + name: {{ include "hugegraph.serverCmpdName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + annotations: + apps.kubeblocks.io/skip-immutable-check: "true" + {{- include "hugegraph.annotations" . | nindent 4 }} +spec: + provider: kubeblocks + description: Apache HugeGraph 1.7.0 query server on HStore. + serviceKind: hugegraph + serviceVersion: {{ .Values.serviceVersion }} + minReadySeconds: 10 + replicasLimit: + minReplicas: 1 + maxReplicas: 5 + services: + - name: default + spec: + ports: + - name: http + port: 8080 + targetPort: http + - name: gremlin + port: 8182 + targetPort: gremlin + scripts: + - name: hugegraph-scripts + template: {{ include "hugegraph.scriptsTemplateName" . }} + namespace: {{ .Release.Namespace }} + volumeName: scripts + defaultMode: 0555 + vars: + - name: PD_POD_FQDNS + valueFrom: + componentVarRef: + compDef: {{ include "hugegraph.pdCmpdPattern" . }} + optional: false + podFQDNs: Required + - name: STORE_POD_FQDNS + valueFrom: + componentVarRef: + compDef: {{ include "hugegraph.storeCmpdPattern" . }} + optional: false + podFQDNs: Required + runtime: + containers: + - name: server + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /bin/bash + - /scripts/start-server.sh + ports: + - name: http + containerPort: 8080 + - name: gremlin + containerPort: 8182 + startupProbe: + httpGet: + path: /versions + port: http + failureThreshold: 36 + periodSeconds: 5 + timeoutSeconds: 3 + readinessProbe: + httpGet: + path: /versions + port: http + failureThreshold: 3 + periodSeconds: 10 + timeoutSeconds: 3 + livenessProbe: + httpGet: + path: /versions + port: http + failureThreshold: 6 + periodSeconds: 15 + timeoutSeconds: 3 + volumeMounts: + - name: scripts + mountPath: /scripts diff --git a/addons/hugegraph/templates/cmpd-store.yaml b/addons/hugegraph/templates/cmpd-store.yaml new file mode 100644 index 0000000000..abce0aa5be --- /dev/null +++ b/addons/hugegraph/templates/cmpd-store.yaml @@ -0,0 +1,100 @@ +apiVersion: apps.kubeblocks.io/v1 +kind: ComponentDefinition +metadata: + name: {{ include "hugegraph.storeCmpdName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + annotations: + apps.kubeblocks.io/skip-immutable-check: "true" + {{- include "hugegraph.annotations" . | nindent 4 }} +spec: + provider: kubeblocks + description: Apache HugeGraph 1.7.0 HStore. + serviceKind: hugegraph-store + serviceVersion: {{ .Values.serviceVersion }} + podManagementPolicy: Parallel + minReadySeconds: 10 + replicasLimit: + minReplicas: 1 + maxReplicas: 5 + services: + - name: default + spec: + ports: + - name: rest + port: 8520 + targetPort: rest + - name: grpc + port: 8500 + targetPort: grpc + - name: raft + port: 8510 + targetPort: raft + scripts: + - name: hugegraph-scripts + template: {{ include "hugegraph.scriptsTemplateName" . }} + namespace: {{ .Release.Namespace }} + volumeName: scripts + defaultMode: 0555 + volumes: + - name: data + vars: + - name: PD_POD_FQDNS + valueFrom: + componentVarRef: + compDef: {{ include "hugegraph.pdCmpdPattern" . }} + optional: false + podFQDNs: Required + - name: STORE_POD_FQDNS + valueFrom: + componentVarRef: + compDef: {{ include "hugegraph.storeCmpdPattern" . }} + optional: false + podFQDNs: Required + runtime: + containers: + - name: store + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /bin/bash + - /scripts/start-store.sh + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: HG_STORE_DATA_PATH + value: {{ .Values.distributed.store.dataMountPath }} + ports: + - name: rest + containerPort: 8520 + - name: grpc + containerPort: 8500 + - name: raft + containerPort: 8510 + startupProbe: + httpGet: + path: /v1/health + port: rest + failureThreshold: 48 + periodSeconds: 5 + timeoutSeconds: 3 + readinessProbe: + httpGet: + path: /v1/health + port: rest + failureThreshold: 3 + periodSeconds: 10 + timeoutSeconds: 3 + livenessProbe: + httpGet: + path: /v1/health + port: rest + failureThreshold: 6 + periodSeconds: 15 + timeoutSeconds: 3 + volumeMounts: + - name: data + mountPath: {{ .Values.distributed.store.dataMountPath }} + - name: scripts + mountPath: /scripts diff --git a/addons/hugegraph/templates/cmpv.yaml b/addons/hugegraph/templates/cmpv.yaml index 276e22fd07..e3d695ee6b 100644 --- a/addons/hugegraph/templates/cmpv.yaml +++ b/addons/hugegraph/templates/cmpv.yaml @@ -12,8 +12,32 @@ spec: - {{ include "hugegraph.cmpdPattern" . }} releases: - hugegraph-1.7.0 + - compDefs: + - {{ include "hugegraph.pdCmpdPattern" . }} + releases: + - hugegraph-pd-1.7.0 + - compDefs: + - {{ include "hugegraph.storeCmpdPattern" . }} + releases: + - hugegraph-store-1.7.0 + - compDefs: + - {{ include "hugegraph.serverCmpdPattern" . }} + releases: + - hugegraph-server-1.7.0 releases: - name: hugegraph-1.7.0 serviceVersion: 1.7.0 images: hugegraph: {{ include "hugegraph.image" . }} + - name: hugegraph-pd-1.7.0 + serviceVersion: 1.7.0 + images: + pd: {{ include "hugegraph.pdImage" . }} + - name: hugegraph-store-1.7.0 + serviceVersion: 1.7.0 + images: + store: {{ include "hugegraph.storeImage" . }} + - name: hugegraph-server-1.7.0 + serviceVersion: 1.7.0 + images: + server: {{ include "hugegraph.serverImage" . }} diff --git a/addons/hugegraph/templates/script-template.yaml b/addons/hugegraph/templates/script-template.yaml index 4216c3be73..7bb8aa6610 100644 --- a/addons/hugegraph/templates/script-template.yaml +++ b/addons/hugegraph/templates/script-template.yaml @@ -9,3 +9,9 @@ data: {{- .Files.Get "scripts/start.sh" | nindent 4 }} shutdown.sh: |- {{- .Files.Get "scripts/shutdown.sh" | nindent 4 }} + start-pd.sh: |- + {{- .Files.Get "scripts/start-pd.sh" | nindent 4 }} + start-store.sh: |- + {{- .Files.Get "scripts/start-store.sh" | nindent 4 }} + start-server.sh: |- + {{- .Files.Get "scripts/start-server.sh" | nindent 4 }} diff --git a/addons/hugegraph/tests/contract_test.sh b/addons/hugegraph/tests/contract_test.sh index b0cf6a2fe7..bbca77f2e9 100644 --- a/addons/hugegraph/tests/contract_test.sh +++ b/addons/hugegraph/tests/contract_test.sh @@ -49,11 +49,18 @@ required_files=( "${ADDON_DIR}/templates/actionset.yaml" "${ADDON_DIR}/templates/script-template.yaml" "${ADDON_DIR}/scripts/start.sh" + "${ADDON_DIR}/scripts/start-pd.sh" + "${ADDON_DIR}/scripts/start-store.sh" + "${ADDON_DIR}/scripts/start-server.sh" "${ADDON_DIR}/scripts/shutdown.sh" "${ADDON_DIR}/scripts/backup.sh" "${ADDON_DIR}/scripts/restore.sh" + "${ADDON_DIR}/templates/cmpd-pd.yaml" + "${ADDON_DIR}/templates/cmpd-store.yaml" + "${ADDON_DIR}/templates/cmpd-server.yaml" "${ADDON_DIR}/tests/scripts_test.sh" "${ADDON_DIR}/exporter/Dockerfile" + "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" "${ADDON_DIR}/exporter/go.mod" "${CLUSTER_DIR}/Chart.yaml" "${CLUSTER_DIR}/templates/cluster.yaml" @@ -68,6 +75,9 @@ done bash -n \ "${ADDON_DIR}/scripts/start.sh" \ + "${ADDON_DIR}/scripts/start-pd.sh" \ + "${ADDON_DIR}/scripts/start-store.sh" \ + "${ADDON_DIR}/scripts/start-server.sh" \ "${ADDON_DIR}/scripts/shutdown.sh" \ "${ADDON_DIR}/scripts/backup.sh" \ "${ADDON_DIR}/scripts/restore.sh" @@ -139,31 +149,31 @@ assert_contains "${definition_render}" 'metricsPath: /metrics' assert_contains "${definition_render}" 'collectionInterval: 30s' assert_equal \ - "$(yq 'select(.kind == "ComponentDefinition") | .spec.runtime.containers | length' "${definition_render}")" \ + "$(yq 'select(.kind == "ComponentDefinition" and .metadata.name == "hugegraph-1.0.0") | .spec.runtime.containers | length' "${definition_render}")" \ "2" \ - "component container count" + "standalone container count" assert_equal \ - "$(yq 'select(.kind == "ComponentDefinition") | .spec.volumes | length' "${definition_render}")" \ + "$(yq 'select(.kind == "ComponentDefinition" and .metadata.name == "hugegraph-1.0.0") | .spec.volumes | length' "${definition_render}")" \ "1" \ - "component volume count" + "standalone volume count" assert_equal \ - "$(yq 'select(.kind == "ComponentDefinition") | .spec.volumes[0].name' "${definition_render}")" \ + "$(yq 'select(.kind == "ComponentDefinition" and .metadata.name == "hugegraph-1.0.0") | .spec.volumes[0].name' "${definition_render}")" \ "data" \ - "component data volume" + "standalone data volume" assert_equal \ - "$(yq 'select(.kind == "ComponentDefinition") | .spec.runtime.containers[] | select(.name == "hugegraph") | .volumeMounts | length' "${definition_render}")" \ + "$(yq 'select(.kind == "ComponentDefinition" and .metadata.name == "hugegraph-1.0.0") | .spec.runtime.containers[] | select(.name == "hugegraph") | .volumeMounts | length' "${definition_render}")" \ "2" \ "main container volume mounts" assert_equal \ - "$(yq 'select(.kind == "ComponentDefinition") | .spec.runtime.containers[] | select(.name == "hugegraph-exporter") | (.volumeMounts // []) | length' "${definition_render}")" \ + "$(yq 'select(.kind == "ComponentDefinition" and .metadata.name == "hugegraph-1.0.0") | .spec.runtime.containers[] | select(.name == "hugegraph-exporter") | (.volumeMounts // []) | length' "${definition_render}")" \ "0" \ "exporter PVC mounts" assert_equal \ - "$(yq 'select(.kind == "ComponentDefinition") | .spec.runtime.containers[] | select(.name == "hugegraph-exporter") | .env | length' "${definition_render}")" \ + "$(yq 'select(.kind == "ComponentDefinition" and .metadata.name == "hugegraph-1.0.0") | .spec.runtime.containers[] | select(.name == "hugegraph-exporter") | .env | length' "${definition_render}")" \ "2" \ "exporter credential env count" assert_equal \ - "$(yq 'select(.kind == "ComponentDefinition") | .spec.exporter.containerName' "${definition_render}")" \ + "$(yq 'select(.kind == "ComponentDefinition" and .metadata.name == "hugegraph-1.0.0") | .spec.exporter.containerName' "${definition_render}")" \ "hugegraph-exporter" \ "spec.exporter container" assert_equal \ @@ -177,6 +187,34 @@ assert_contains "${cluster_render}" 'clusterDef: hugegraph' assert_contains "${cluster_render}" 'topology: standalone' assert_contains "${cluster_render}" 'replicas: 1' assert_contains "${cluster_render}" 'serviceVersion: [\"]?1\.7\.0[\"]?' + +assert_contains "${definition_render}" 'name: distributed' +assert_contains "${definition_render}" 'name: hugegraph-pd-1.0.0' +assert_contains "${definition_render}" 'name: hugegraph-store-1.0.0' +assert_contains "${definition_render}" 'name: hugegraph-server-1.0.0' +assert_contains "${definition_render}" 'compDef: \^hugegraph-\[0-9\]' +assert_contains "${definition_render}" 'compDef: \^hugegraph-pd-' +assert_contains "${definition_render}" 'docker.io/hugegraph/pd:1.7.0' +assert_contains "${definition_render}" 'docker.io/hugegraph/store:1.7.0' +assert_contains "${definition_render}" 'docker.io/hugegraph/server:1.7.0' +assert_contains "${ADDON_DIR}/scripts/start-pd.sh" 'HG_PD_RAFT_PEERS_LIST' +assert_contains "${ADDON_DIR}/scripts/start-pd.sh" 'HG_PD_RAFT_ADDRESS' +assert_contains "${ADDON_DIR}/scripts/start-store.sh" 'HG_STORE_PD_ADDRESS' +assert_contains "${ADDON_DIR}/scripts/start-server.sh" 'HG_SERVER_BACKEND=hstore' +assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'topology: distributed' +assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'name: pd' +assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'name: store' + +distributed_render=$(mktemp) +helm template hugegraph "${CLUSTER_DIR}" --namespace demo --set topology=distributed >"${distributed_render}" +assert_contains "${distributed_render}" 'topology: distributed' +assert_contains "${distributed_render}" 'name: pd' +assert_contains "${distributed_render}" 'name: store' +assert_equal \ + "$(yq '[select(.kind == "Cluster") | .spec.componentSpecs[].name] | join(",")' "${distributed_render}")" \ + "pd,store,server" \ + "distributed component names" +rm -f "${distributed_render}" assert_contains "${ADDON_DIR}/.helmignore" '^exporter/' package_dir=$(mktemp -d) diff --git a/addons/hugegraph/values.yaml b/addons/hugegraph/values.yaml index de9e24c0e6..d907ea3b0b 100644 --- a/addons/hugegraph/values.yaml +++ b/addons/hugegraph/values.yaml @@ -21,6 +21,19 @@ exporter: cpu: 100m memory: 128Mi +distributed: + pd: + repository: hugegraph/pd + tag: "1.7.0" + dataMountPath: /hugegraph-pd/pd_data + store: + repository: hugegraph/store + tag: "1.7.0" + dataMountPath: /hugegraph-store/storage + server: + repository: hugegraph/server + tag: "1.7.0" + serviceVersion: "1.7.0" dataMountPath: /hugegraph-data diff --git a/examples/hugegraph/README.md b/examples/hugegraph/README.md index 2ea441b18d..c9790f8795 100644 --- a/examples/hugegraph/README.md +++ b/examples/hugegraph/README.md @@ -1,6 +1,7 @@ # HugeGraph Examples -These examples target KubeBlocks `release-1.0` and HugeGraph 1.7.0 standalone. +These examples target KubeBlocks `release-1.0` and HugeGraph 1.7.0. +`cluster.yaml` is standalone. `cluster-distributed.yaml` is PD + Store + Server. ## Create diff --git a/examples/hugegraph/cluster-distributed.yaml b/examples/hugegraph/cluster-distributed.yaml new file mode 100644 index 0000000000..9ee2b25dd0 --- /dev/null +++ b/examples/hugegraph/cluster-distributed.yaml @@ -0,0 +1,58 @@ +apiVersion: apps.kubeblocks.io/v1 +kind: Cluster +metadata: + name: hugegraph-distributed + namespace: demo +spec: + clusterDef: hugegraph + topology: distributed + terminationPolicy: Delete + componentSpecs: + - name: pd + serviceVersion: "1.7.0" + replicas: 3 + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "500m" + memory: 1Gi + volumeClaimTemplates: + - name: data + spec: + storageClassName: "" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + - name: store + serviceVersion: "1.7.0" + replicas: 3 + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "500m" + memory: 1Gi + volumeClaimTemplates: + - name: data + spec: + storageClassName: "" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + - name: server + serviceVersion: "1.7.0" + replicas: 1 + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "500m" + memory: 1Gi From 754d5d61dfbe863f3e2e9b1fc57ce97e134750d0 Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Wed, 19 Aug 2026 02:20:08 +0800 Subject: [PATCH 06/16] docs(hugegraph): declare distributed limits in README (#3415) Co-authored-by: Amber --- addons/hugegraph/DESIGN.zh.md | 2 +- addons/hugegraph/README.md | 54 ++++++++++++++----------- addons/hugegraph/tests/contract_test.sh | 4 ++ 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/addons/hugegraph/DESIGN.zh.md b/addons/hugegraph/DESIGN.zh.md index 95cb7e8f04..b8d7ea4bd7 100644 --- a/addons/hugegraph/DESIGN.zh.md +++ b/addons/hugegraph/DESIGN.zh.md @@ -38,7 +38,7 @@ PVC。 - 多 graph 之间的全局事务时间点。每个 graph 自身 checkpoint 一致,但 graph 之间 按顺序创建 checkpoint,存在短时间窗口。 - RebuildInstance、scaleOut.fromBackup、跨 HugeGraph 版本、跨 topology 恢复。 -- TLS、在线参数变更、分布式 PD/Store/Server topology。 +- TLS、在线参数变更。分布式 PD/Store/Server 见 `DESIGN-distributed.zh.md`;本文件只覆盖 standalone。 ## 3. 存储和启动 diff --git a/addons/hugegraph/README.md b/addons/hugegraph/README.md index 3121d6765a..0d29003c26 100644 --- a/addons/hugegraph/README.md +++ b/addons/hugegraph/README.md @@ -1,23 +1,24 @@ # HugeGraph -This addon runs Apache HugeGraph 1.7.0 as a single HugeGraph Server backed by -RocksDB. +This addon runs Apache HugeGraph 1.7.0. Default topology `standalone` is one +Server with embedded RocksDB. Topology `distributed` is PD + Store + Server. ## Capabilities -| Capability | Standalone | -| --- | --- | -| Replicas | Exactly 1 | -| Persistent data | Yes | -| Restart | Yes | -| Stop/Start | Yes | -| Vertical scaling | Yes | -| Volume expansion | Yes, when the StorageClass supports expansion | -| Expose | Yes | -| Full backup/restore | RocksDB checkpoint | -| Horizontal scaling | No | -| Reconfigure | No | -| TLS | No | +| Capability | Standalone | Distributed | +| --- | --- | --- | +| Replicas | Exactly 1 server | pd 3, store 3, server 1 | +| Persistent data | Yes | PD and Store data PVCs | +| Restart | Yes | Yes | +| Stop/Start | Yes | Yes | +| Vertical scaling | Yes | Yes | +| Volume expansion | Yes, when the StorageClass supports expansion | PD/Store only, when the StorageClass supports expansion | +| Expose | Yes | Yes | +| Full backup/restore | RocksDB checkpoint | No | +| Horizontal scaling | No | No. Store scale-in/rebalance is not supported | +| roleProbe / switchover | No | No | +| Reconfigure | No | No | +| TLS | No | No | The `checkpoint` backup method calls HugeGraph's `snapshot_create` API for every persistent graph. It uploads graph configurations, a format-versioned @@ -25,11 +26,14 @@ manifest, SHA-256 checksums, and all RocksDB checkpoints through datasafed. It does not use CSI volume snapshots. Restore is supported only to a new HugeGraph 1.7.0 standalone Cluster. It -restores the complete instance. Single-graph restore, PITR, incremental backup, -RebuildInstance, cross-version restore, and cross-topology restore are not -supported. The restore action validates every graph config and checkpoint file, -then creates the persistent upstream initialization marker before the first -HugeGraph process starts. +restores the complete instance. Distributed backup/restore, single-graph +restore, PITR, incremental backup, RebuildInstance, cross-version restore, and +cross-topology restore are not supported. The restore action validates every +graph config and checkpoint file, then creates the persistent upstream +initialization marker before the first HugeGraph process starts. + +`BackupPolicyTemplate` matches only the standalone Server ComponentDefinition. +Do not treat a distributed Cluster as backup-capable. ## Storage @@ -66,10 +70,12 @@ transaction timestamp. ## Image -The runtime and data protection actions use -`docker.io/hugegraph/hugegraph:1.7.0`. No custom tools image is required. +Standalone uses `docker.io/hugegraph/hugegraph:1.7.0`. Distributed uses +`hugegraph/pd:1.7.0`, `hugegraph/store:1.7.0`, and `hugegraph/server:1.7.0`. +No custom tools image is required. Chart CI rewrites those official names to +`docker.io/apecloud/*` and requires amd64+arm64 indexes. ## Examples -See `examples/hugegraph` for Cluster, backup, restore, restart, stop/start, -vertical scaling, and volume expansion manifests. +See `examples/hugegraph` for standalone Cluster, backup, restore, restart, +stop/start, vertical scaling, volume expansion, and `cluster-distributed.yaml`. diff --git a/addons/hugegraph/tests/contract_test.sh b/addons/hugegraph/tests/contract_test.sh index bbca77f2e9..cf04fecd09 100644 --- a/addons/hugegraph/tests/contract_test.sh +++ b/addons/hugegraph/tests/contract_test.sh @@ -204,6 +204,10 @@ assert_contains "${ADDON_DIR}/scripts/start-server.sh" 'HG_SERVER_BACKEND=hstore assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'topology: distributed' assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'name: pd' assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'name: store' +assert_contains "${ADDON_DIR}/README.md" 'Topology `distributed`' +assert_contains "${ADDON_DIR}/README.md" 'Distributed backup/restore' +assert_contains "${ADDON_DIR}/README.md" 'Store scale-in/rebalance is not supported' +assert_contains "${ADDON_DIR}/README.md" 'matches only the standalone Server ComponentDefinition' distributed_render=$(mktemp) helm template hugegraph "${CLUSTER_DIR}" --namespace demo --set topology=distributed >"${distributed_render}" From 83540814afa861acc92eeaa5277093a668ef8160 Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Wed, 19 Aug 2026 02:47:51 +0800 Subject: [PATCH 07/16] docs(hugegraph): mark distributed systemAccounts unsupported (#3416) Co-authored-by: Amber --- addons/hugegraph/DESIGN-distributed.zh.md | 1 + addons/hugegraph/README.md | 4 ++++ addons/hugegraph/tests/contract_test.sh | 10 ++++++++++ 3 files changed, 15 insertions(+) diff --git a/addons/hugegraph/DESIGN-distributed.zh.md b/addons/hugegraph/DESIGN-distributed.zh.md index 0069c0c41d..d6c3209d14 100644 --- a/addons/hugegraph/DESIGN-distributed.zh.md +++ b/addons/hugegraph/DESIGN-distributed.zh.md @@ -58,5 +58,6 @@ headless Service 名。 - PD / Store switchover、roleProbe - TLS、reconfigure - Hubble +- distributed `systemAccounts` / 生成 admin Secret BackupPolicyTemplate 继续只命中 standalone server CmpD。 diff --git a/addons/hugegraph/README.md b/addons/hugegraph/README.md index 0d29003c26..dfabf0c5b8 100644 --- a/addons/hugegraph/README.md +++ b/addons/hugegraph/README.md @@ -15,6 +15,7 @@ Server with embedded RocksDB. Topology `distributed` is PD + Store + Server. | Volume expansion | Yes, when the StorageClass supports expansion | PD/Store only, when the StorageClass supports expansion | | Expose | Yes | Yes | | Full backup/restore | RocksDB checkpoint | No | +| System account | `admin` init Secret | No | | Horizontal scaling | No | No. Store scale-in/rebalance is not supported | | roleProbe / switchover | No | No | | Reconfigure | No | No | @@ -35,6 +36,9 @@ initialization marker before the first HugeGraph process starts. `BackupPolicyTemplate` matches only the standalone Server ComponentDefinition. Do not treat a distributed Cluster as backup-capable. +Distributed PD, Store, and Server do not declare `systemAccounts`. There is +no generated admin Secret for `topology=distributed`. + ## Storage The `data` PVC is mounted at `/hugegraph-data`. The addon keeps graph diff --git a/addons/hugegraph/tests/contract_test.sh b/addons/hugegraph/tests/contract_test.sh index cf04fecd09..55ab58b129 100644 --- a/addons/hugegraph/tests/contract_test.sh +++ b/addons/hugegraph/tests/contract_test.sh @@ -208,6 +208,16 @@ assert_contains "${ADDON_DIR}/README.md" 'Topology `distributed`' assert_contains "${ADDON_DIR}/README.md" 'Distributed backup/restore' assert_contains "${ADDON_DIR}/README.md" 'Store scale-in/rebalance is not supported' assert_contains "${ADDON_DIR}/README.md" 'matches only the standalone Server ComponentDefinition' +assert_contains "${ADDON_DIR}/README.md" 'do not declare `systemAccounts`' +assert_contains "${ADDON_DIR}/README.md" 'no generated admin Secret' +assert_equal \ + "$(yq '[select(.kind == "BackupPolicyTemplate") | .spec.compDefs[]] | join(",")' "${definition_render}" | tr -d '\n')" \ + "^hugegraph-[0-9]" \ + "BPT matches only standalone server" +assert_equal \ + "$(yq ea '[select(.kind == "ComponentDefinition" and (.metadata.name == "hugegraph-pd-1.0.0" or .metadata.name == "hugegraph-store-1.0.0" or .metadata.name == "hugegraph-server-1.0.0")) | .spec | has("systemAccounts")] | map(select(. == true)) | length' "${definition_render}" | tr -d '\n')" \ + "0" \ + "distributed CmpDs have no systemAccounts" distributed_render=$(mktemp) helm template hugegraph "${CLUSTER_DIR}" --namespace demo --set topology=distributed >"${distributed_render}" From f79df4529a9e48861cd191f1da7659adbbadb1b4 Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Wed, 19 Aug 2026 09:25:56 +0800 Subject: [PATCH 08/16] fix(hugegraph): fail store start if PD health never succeeds (#3417) Co-authored-by: Amber --- addons/hugegraph/scripts/start-store.sh | 12 ++++- addons/hugegraph/tests/contract_test.sh | 3 ++ addons/hugegraph/tests/start_store_test.sh | 62 ++++++++++++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) create mode 100755 addons/hugegraph/tests/start_store_test.sh diff --git a/addons/hugegraph/scripts/start-store.sh b/addons/hugegraph/scripts/start-store.sh index cad0bf17bf..14695ef5b9 100644 --- a/addons/hugegraph/scripts/start-store.sh +++ b/addons/hugegraph/scripts/start-store.sh @@ -35,12 +35,20 @@ done export HG_STORE_PD_ADDRESS="$(append_port "${PD_POD_FQDNS}" 8686)" first_pd=${PD_POD_FQDNS%%,*} -for _ in $(seq 1 60); do +attempts=${HG_STORE_HEALTH_ATTEMPTS:-60} +sleep_secs=${HG_STORE_HEALTH_SLEEP:-2} +pd_healthy=0 +for _ in $(seq 1 "${attempts}"); do if curl -fsS "http://${first_pd}:8620/v1/health" >/dev/null; then + pd_healthy=1 break fi - sleep 2 + sleep "${sleep_secs}" done +[[ "${pd_healthy}" -eq 1 ]] || { + echo "PD is not healthy at http://${first_pd}:8620/v1/health after ${attempts} attempt(s)" >&2 + exit 1 +} export HG_STORE_GRPC_HOST="${self}" export HG_STORE_GRPC_PORT="${HG_STORE_GRPC_PORT:-8500}" export HG_STORE_REST_PORT="${HG_STORE_REST_PORT:-8520}" diff --git a/addons/hugegraph/tests/contract_test.sh b/addons/hugegraph/tests/contract_test.sh index 55ab58b129..16f662c164 100644 --- a/addons/hugegraph/tests/contract_test.sh +++ b/addons/hugegraph/tests/contract_test.sh @@ -59,6 +59,7 @@ required_files=( "${ADDON_DIR}/templates/cmpd-store.yaml" "${ADDON_DIR}/templates/cmpd-server.yaml" "${ADDON_DIR}/tests/scripts_test.sh" + "${ADDON_DIR}/tests/start_store_test.sh" "${ADDON_DIR}/exporter/Dockerfile" "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" "${ADDON_DIR}/exporter/go.mod" @@ -200,6 +201,7 @@ assert_contains "${definition_render}" 'docker.io/hugegraph/server:1.7.0' assert_contains "${ADDON_DIR}/scripts/start-pd.sh" 'HG_PD_RAFT_PEERS_LIST' assert_contains "${ADDON_DIR}/scripts/start-pd.sh" 'HG_PD_RAFT_ADDRESS' assert_contains "${ADDON_DIR}/scripts/start-store.sh" 'HG_STORE_PD_ADDRESS' +assert_contains "${ADDON_DIR}/scripts/start-store.sh" 'PD is not healthy' assert_contains "${ADDON_DIR}/scripts/start-server.sh" 'HG_SERVER_BACKEND=hstore' assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'topology: distributed' assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'name: pd' @@ -243,5 +245,6 @@ package_bytes=$(wc -c < "${package_tgz}" | tr -d ' ') [[ "${package_bytes}" -lt 100000 ]] || fail "packaged chart too large: ${package_bytes} bytes" "${ADDON_DIR}/tests/scripts_test.sh" +"${ADDON_DIR}/tests/start_store_test.sh" echo "HugeGraph addon offline contracts passed" diff --git a/addons/hugegraph/tests/start_store_test.sh b/addons/hugegraph/tests/start_store_test.sh new file mode 100755 index 0000000000..fdb66bb8f9 --- /dev/null +++ b/addons/hugegraph/tests/start_store_test.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +TEST_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ADDON_DIR=$(cd "${TEST_DIR}/.." && pwd) +WORK_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/hugegraph-start-store-test.XXXXXX") +MOCK_BIN="${WORK_ROOT}/bin" + +cleanup() { + rm -rf -- "$WORK_ROOT" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +mkdir -p "$MOCK_BIN" + +cat >"${MOCK_BIN}/curl" <<'MOCK' +#!/usr/bin/env bash +exit 1 +MOCK +chmod +x "${MOCK_BIN}/curl" + +cat >"${MOCK_BIN}/dumb-init" <<'MOCK' +#!/usr/bin/env bash +echo "dumb-init should not run" >&2 +exit 99 +MOCK +chmod +x "${MOCK_BIN}/dumb-init" + +mkdir -p "${WORK_ROOT}/hugegraph-store" +printf '#!/usr/bin/env bash\necho entrypoint should not run\nexit 98\n' \ + >"${WORK_ROOT}/hugegraph-store/docker-entrypoint.sh" +chmod +x "${WORK_ROOT}/hugegraph-store/docker-entrypoint.sh" + +export PATH="${MOCK_BIN}:/usr/bin:/bin" +export PD_POD_FQDNS=pd-0.pd-headless +export STORE_POD_FQDNS=store-0.store-headless +export POD_NAME=store-0 +export HG_STORE_HEALTH_ATTEMPTS=2 +export HG_STORE_HEALTH_SLEEP=0 +export HG_STORE_DATA_PATH="${WORK_ROOT}/store-data" + +set +e +( + cd "$WORK_ROOT" + bash "${ADDON_DIR}/scripts/start-store.sh" +) >"${WORK_ROOT}/out" 2>"${WORK_ROOT}/err" +rc=$? +set -e + +[[ "$rc" -ne 0 ]] || fail "start-store.sh continued after PD health never succeeded" +if grep -q 'should not run' "${WORK_ROOT}/out" "${WORK_ROOT}/err"; then + fail "start-store.sh reached entrypoint after PD health failed" +fi +grep -q 'PD is not healthy' "${WORK_ROOT}/err" \ + || fail "start-store.sh did not report PD health failure" + +echo "HugeGraph start-store PD wait tests passed" From bb20a02c2e9d17143ed5ed02f9682e0b8740ddaa Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Wed, 19 Aug 2026 10:28:27 +0800 Subject: [PATCH 09/16] fix(hugegraph): fail server start if Store health never succeeds (#3418) Co-authored-by: Amber --- addons/hugegraph/scripts/start-server.sh | 15 ++++++ addons/hugegraph/tests/contract_test.sh | 3 ++ addons/hugegraph/tests/start_server_test.sh | 55 +++++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100755 addons/hugegraph/tests/start_server_test.sh diff --git a/addons/hugegraph/scripts/start-server.sh b/addons/hugegraph/scripts/start-server.sh index b0fb6b00f3..a7e1db8d1b 100644 --- a/addons/hugegraph/scripts/start-server.sh +++ b/addons/hugegraph/scripts/start-server.sh @@ -25,5 +25,20 @@ export HG_SERVER_USE_PD=true export HG_SERVER_INIT_STORE_ENABLED=false export STORE_REST="${first_store}:8520" +attempts=${HG_SERVER_HEALTH_ATTEMPTS:-60} +sleep_secs=${HG_SERVER_HEALTH_SLEEP:-2} +store_healthy=0 +for _ in $(seq 1 "${attempts}"); do + if curl -fsS "http://${first_store}:8520/v1/health" >/dev/null; then + store_healthy=1 + break + fi + sleep "${sleep_secs}" +done +[[ "${store_healthy}" -eq 1 ]] || { + echo "Store is not healthy at http://${first_store}:8520/v1/health after ${attempts} attempt(s)" >&2 + exit 1 +} + cd /hugegraph-server exec /usr/bin/dumb-init -- ./docker-entrypoint.sh diff --git a/addons/hugegraph/tests/contract_test.sh b/addons/hugegraph/tests/contract_test.sh index 16f662c164..abd27df3b6 100644 --- a/addons/hugegraph/tests/contract_test.sh +++ b/addons/hugegraph/tests/contract_test.sh @@ -60,6 +60,7 @@ required_files=( "${ADDON_DIR}/templates/cmpd-server.yaml" "${ADDON_DIR}/tests/scripts_test.sh" "${ADDON_DIR}/tests/start_store_test.sh" + "${ADDON_DIR}/tests/start_server_test.sh" "${ADDON_DIR}/exporter/Dockerfile" "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" "${ADDON_DIR}/exporter/go.mod" @@ -203,6 +204,7 @@ assert_contains "${ADDON_DIR}/scripts/start-pd.sh" 'HG_PD_RAFT_ADDRESS' assert_contains "${ADDON_DIR}/scripts/start-store.sh" 'HG_STORE_PD_ADDRESS' assert_contains "${ADDON_DIR}/scripts/start-store.sh" 'PD is not healthy' assert_contains "${ADDON_DIR}/scripts/start-server.sh" 'HG_SERVER_BACKEND=hstore' +assert_contains "${ADDON_DIR}/scripts/start-server.sh" 'Store is not healthy' assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'topology: distributed' assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'name: pd' assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'name: store' @@ -246,5 +248,6 @@ package_bytes=$(wc -c < "${package_tgz}" | tr -d ' ') "${ADDON_DIR}/tests/scripts_test.sh" "${ADDON_DIR}/tests/start_store_test.sh" +"${ADDON_DIR}/tests/start_server_test.sh" echo "HugeGraph addon offline contracts passed" diff --git a/addons/hugegraph/tests/start_server_test.sh b/addons/hugegraph/tests/start_server_test.sh new file mode 100755 index 0000000000..13aecbeefe --- /dev/null +++ b/addons/hugegraph/tests/start_server_test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +TEST_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ADDON_DIR=$(cd "${TEST_DIR}/.." && pwd) +WORK_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/hugegraph-start-server-test.XXXXXX") +MOCK_BIN="${WORK_ROOT}/bin" + +cleanup() { + rm -rf -- "$WORK_ROOT" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +mkdir -p "$MOCK_BIN" + +cat >"${MOCK_BIN}/curl" <<'MOCK' +#!/usr/bin/env bash +exit 1 +MOCK +chmod +x "${MOCK_BIN}/curl" + +cat >"${MOCK_BIN}/dumb-init" <<'MOCK' +#!/usr/bin/env bash +echo "dumb-init should not run" >&2 +exit 99 +MOCK +chmod +x "${MOCK_BIN}/dumb-init" + +export PATH="${MOCK_BIN}:/usr/bin:/bin" +export PD_POD_FQDNS=pd-0.pd-headless +export STORE_POD_FQDNS=store-0.store-headless +export HG_SERVER_HEALTH_ATTEMPTS=2 +export HG_SERVER_HEALTH_SLEEP=0 + +set +e +( + cd "$WORK_ROOT" + bash "${ADDON_DIR}/scripts/start-server.sh" +) >"${WORK_ROOT}/out" 2>"${WORK_ROOT}/err" +rc=$? +set -e + +[[ "$rc" -ne 0 ]] || fail "start-server.sh continued after Store health never succeeded" +if grep -q 'should not run' "${WORK_ROOT}/out" "${WORK_ROOT}/err"; then + fail "start-server.sh reached entrypoint after Store health failed" +fi +grep -q 'Store is not healthy' "${WORK_ROOT}/err" \ + || fail "start-server.sh did not report Store health failure" + +echo "HugeGraph start-server Store wait tests passed" From 9b1cc331d40b8a736c9d4fed68271cf82017ab41 Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Wed, 19 Aug 2026 12:18:05 +0800 Subject: [PATCH 10/16] fix(hugegraph): derive PD initial store count from store FQDNs (#3419) Co-authored-by: Amber --- addons/hugegraph/scripts/start-pd.sh | 29 +++++++++++- addons/hugegraph/tests/contract_test.sh | 4 ++ addons/hugegraph/tests/start_pd_test.sh | 61 +++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100755 addons/hugegraph/tests/start_pd_test.sh diff --git a/addons/hugegraph/scripts/start-pd.sh b/addons/hugegraph/scripts/start-pd.sh index 2f4d3a4489..1fe1000727 100644 --- a/addons/hugegraph/scripts/start-pd.sh +++ b/addons/hugegraph/scripts/start-pd.sh @@ -19,6 +19,17 @@ append_port() { printf '%s' "${out}" } +count_hosts() { + local list=$1 + local n=0 host + IFS=',' read -ra hosts <<< "${list}" + for host in "${hosts[@]}"; do + [[ -n "${host}" ]] || continue + n=$((n + 1)) + done + printf '%s' "${n}" +} + self="" IFS=',' read -ra hosts <<< "${PD_POD_FQDNS}" for host in "${hosts[@]}"; do @@ -33,6 +44,12 @@ done exit 1 } +store_count="$(count_hosts "${STORE_POD_FQDNS}")" +[[ "${store_count}" -ge 1 ]] || { + echo "cannot derive store count from STORE_POD_FQDNS=${STORE_POD_FQDNS}" >&2 + exit 1 +} + export HG_PD_GRPC_HOST="${self}" export HG_PD_GRPC_PORT="${HG_PD_GRPC_PORT:-8686}" export HG_PD_REST_PORT="${HG_PD_REST_PORT:-8620}" @@ -40,7 +57,17 @@ export HG_PD_RAFT_ADDRESS="${self}:8610" export HG_PD_RAFT_PEERS_LIST="$(append_port "${PD_POD_FQDNS}" 8610)" export HG_PD_INITIAL_STORE_LIST="$(append_port "${STORE_POD_FQDNS}" 8500)" export HG_PD_DATA_PATH="${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data}" -export HG_PD_INITIAL_STORE_COUNT="${HG_PD_INITIAL_STORE_COUNT:-1}" +# Official pd.initial-store-count must match the expected store count +# (3 for 3 stores). Defaulting to 1 activates the cluster on the first +# Store and can leave the remaining members out of the first partition +# allocation. +export HG_PD_INITIAL_STORE_COUNT="${HG_PD_INITIAL_STORE_COUNT:-${store_count}}" + +if [[ "${HG_PD_DRY_RUN:-0}" == "1" ]]; then + printf 'HG_PD_INITIAL_STORE_COUNT=%s\n' "${HG_PD_INITIAL_STORE_COUNT}" + printf 'HG_PD_INITIAL_STORE_LIST=%s\n' "${HG_PD_INITIAL_STORE_LIST}" + exit 0 +fi mkdir -p "${HG_PD_DATA_PATH}" cd /hugegraph-pd diff --git a/addons/hugegraph/tests/contract_test.sh b/addons/hugegraph/tests/contract_test.sh index abd27df3b6..a79211fecd 100644 --- a/addons/hugegraph/tests/contract_test.sh +++ b/addons/hugegraph/tests/contract_test.sh @@ -59,6 +59,7 @@ required_files=( "${ADDON_DIR}/templates/cmpd-store.yaml" "${ADDON_DIR}/templates/cmpd-server.yaml" "${ADDON_DIR}/tests/scripts_test.sh" + "${ADDON_DIR}/tests/start_pd_test.sh" "${ADDON_DIR}/tests/start_store_test.sh" "${ADDON_DIR}/tests/start_server_test.sh" "${ADDON_DIR}/exporter/Dockerfile" @@ -201,6 +202,8 @@ assert_contains "${definition_render}" 'docker.io/hugegraph/store:1.7.0' assert_contains "${definition_render}" 'docker.io/hugegraph/server:1.7.0' assert_contains "${ADDON_DIR}/scripts/start-pd.sh" 'HG_PD_RAFT_PEERS_LIST' assert_contains "${ADDON_DIR}/scripts/start-pd.sh" 'HG_PD_RAFT_ADDRESS' +assert_contains "${ADDON_DIR}/scripts/start-pd.sh" 'HG_PD_INITIAL_STORE_COUNT=' +assert_contains "${ADDON_DIR}/scripts/start-pd.sh" 'count_hosts' assert_contains "${ADDON_DIR}/scripts/start-store.sh" 'HG_STORE_PD_ADDRESS' assert_contains "${ADDON_DIR}/scripts/start-store.sh" 'PD is not healthy' assert_contains "${ADDON_DIR}/scripts/start-server.sh" 'HG_SERVER_BACKEND=hstore' @@ -247,6 +250,7 @@ package_bytes=$(wc -c < "${package_tgz}" | tr -d ' ') [[ "${package_bytes}" -lt 100000 ]] || fail "packaged chart too large: ${package_bytes} bytes" "${ADDON_DIR}/tests/scripts_test.sh" +"${ADDON_DIR}/tests/start_pd_test.sh" "${ADDON_DIR}/tests/start_store_test.sh" "${ADDON_DIR}/tests/start_server_test.sh" diff --git a/addons/hugegraph/tests/start_pd_test.sh b/addons/hugegraph/tests/start_pd_test.sh new file mode 100755 index 0000000000..f97c2e6ce7 --- /dev/null +++ b/addons/hugegraph/tests/start_pd_test.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +TEST_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ADDON_DIR=$(cd "${TEST_DIR}/.." && pwd) +WORK_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/hugegraph-start-pd-test.XXXXXX") + +cleanup() { + rm -rf -- "$WORK_ROOT" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +run_start_pd() { + local out=$1 + local err=$2 + ( + export HG_PD_DRY_RUN=1 + export HG_PD_DATA_PATH="${WORK_ROOT}/pd-data" + export PD_POD_FQDNS STORE_POD_FQDNS POD_NAME + if [ -n "${HG_PD_INITIAL_STORE_COUNT+x}" ]; then + export HG_PD_INITIAL_STORE_COUNT + fi + bash "${ADDON_DIR}/scripts/start-pd.sh" + ) >"${out}" 2>"${err}" +} + +# 3-store list must set initial-store-count to 3 (official 3x3 bootstrap). +unset HG_PD_INITIAL_STORE_COUNT || true +PD_POD_FQDNS=pd-0.pd-headless,pd-1.pd-headless,pd-2.pd-headless \ +STORE_POD_FQDNS=store-0.store-headless,store-1.store-headless,store-2.store-headless \ +POD_NAME=pd-0 \ +run_start_pd "${WORK_ROOT}/out3" "${WORK_ROOT}/err3" +grep -qx 'HG_PD_INITIAL_STORE_COUNT=3' "${WORK_ROOT}/out3" \ + || fail "3-store FQDNs did not yield HG_PD_INITIAL_STORE_COUNT=3: $(cat "${WORK_ROOT}/out3" "${WORK_ROOT}/err3")" +grep -q 'HG_PD_INITIAL_STORE_LIST=store-0.store-headless:8500,store-1.store-headless:8500,store-2.store-headless:8500' \ + "${WORK_ROOT}/out3" \ + || fail "3-store FQDNs did not build the store list" + +# 1-store list must stay at 1. +PD_POD_FQDNS=pd-0.pd-headless \ +STORE_POD_FQDNS=store-0.store-headless \ +POD_NAME=pd-0 \ +run_start_pd "${WORK_ROOT}/out1" "${WORK_ROOT}/err1" +grep -qx 'HG_PD_INITIAL_STORE_COUNT=1' "${WORK_ROOT}/out1" \ + || fail "1-store FQDNs did not yield HG_PD_INITIAL_STORE_COUNT=1" + +# Explicit override wins. +PD_POD_FQDNS=pd-0.pd-headless,pd-1.pd-headless,pd-2.pd-headless \ +STORE_POD_FQDNS=store-0.store-headless,store-1.store-headless,store-2.store-headless \ +POD_NAME=pd-0 \ +HG_PD_INITIAL_STORE_COUNT=5 \ +run_start_pd "${WORK_ROOT}/out5" "${WORK_ROOT}/err5" +grep -qx 'HG_PD_INITIAL_STORE_COUNT=5' "${WORK_ROOT}/out5" \ + || fail "explicit HG_PD_INITIAL_STORE_COUNT=5 was not honored" + +echo "HugeGraph start-pd initial store count tests passed" From 2b088d886842173a8251262fedd2729d6c20a944 Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Wed, 19 Aug 2026 12:31:58 +0800 Subject: [PATCH 11/16] fix(hugegraph): wait for every Store before server start (#3420) Co-authored-by: Amber --- addons/hugegraph/scripts/start-server.sh | 38 ++++++++++--- addons/hugegraph/tests/start_server_test.sh | 60 ++++++++++++++++++--- 2 files changed, 82 insertions(+), 16 deletions(-) diff --git a/addons/hugegraph/scripts/start-server.sh b/addons/hugegraph/scripts/start-server.sh index a7e1db8d1b..3253269cf3 100644 --- a/addons/hugegraph/scripts/start-server.sh +++ b/addons/hugegraph/scripts/start-server.sh @@ -17,7 +17,17 @@ append_port() { printf '%s' "${out}" } -first_store=${STORE_POD_FQDNS%%,*} +stores=() +IFS=',' read -ra hosts <<< "${STORE_POD_FQDNS}" +for host in "${hosts[@]}"; do + [[ -n "${host}" ]] || continue + stores+=("${host}") +done +[[ "${#stores[@]}" -ge 1 ]] || { + echo "cannot derive Store list from STORE_POD_FQDNS=${STORE_POD_FQDNS}" >&2 + exit 1 +} +first_store=${stores[0]} export HG_SERVER_BACKEND=hstore export HG_SERVER_PD_PEERS="$(append_port "${PD_POD_FQDNS}" 8686)" @@ -27,18 +37,30 @@ export STORE_REST="${first_store}:8520" attempts=${HG_SERVER_HEALTH_ATTEMPTS:-60} sleep_secs=${HG_SERVER_HEALTH_SLEEP:-2} -store_healthy=0 +# Official 3-store compose waits for every Store, not only the first. +# With pd.initial-store-count = store count, one healthy Store is not +# enough for Cluster_OK. +unhealthy="" for _ in $(seq 1 "${attempts}"); do - if curl -fsS "http://${first_store}:8520/v1/health" >/dev/null; then - store_healthy=1 - break - fi + unhealthy="" + for store in "${stores[@]}"; do + if ! curl -fsS "http://${store}:8520/v1/health" >/dev/null; then + unhealthy=${store} + break + fi + done + [[ -z "${unhealthy}" ]] && break sleep "${sleep_secs}" done -[[ "${store_healthy}" -eq 1 ]] || { - echo "Store is not healthy at http://${first_store}:8520/v1/health after ${attempts} attempt(s)" >&2 +[[ -z "${unhealthy}" ]] || { + echo "Store is not healthy at http://${unhealthy}:8520/v1/health after ${attempts} attempt(s)" >&2 exit 1 } +if [[ "${HG_SERVER_DRY_RUN:-0}" == "1" ]]; then + printf 'stores_healthy=%s\n' "${#stores[@]}" + exit 0 +fi + cd /hugegraph-server exec /usr/bin/dumb-init -- ./docker-entrypoint.sh diff --git a/addons/hugegraph/tests/start_server_test.sh b/addons/hugegraph/tests/start_server_test.sh index 13aecbeefe..566ae7bf3a 100755 --- a/addons/hugegraph/tests/start_server_test.sh +++ b/addons/hugegraph/tests/start_server_test.sh @@ -18,12 +18,6 @@ fail() { mkdir -p "$MOCK_BIN" -cat >"${MOCK_BIN}/curl" <<'MOCK' -#!/usr/bin/env bash -exit 1 -MOCK -chmod +x "${MOCK_BIN}/curl" - cat >"${MOCK_BIN}/dumb-init" <<'MOCK' #!/usr/bin/env bash echo "dumb-init should not run" >&2 @@ -33,10 +27,18 @@ chmod +x "${MOCK_BIN}/dumb-init" export PATH="${MOCK_BIN}:/usr/bin:/bin" export PD_POD_FQDNS=pd-0.pd-headless -export STORE_POD_FQDNS=store-0.store-headless export HG_SERVER_HEALTH_ATTEMPTS=2 export HG_SERVER_HEALTH_SLEEP=0 +# 1) Every Store down — fail closed. +cat >"${MOCK_BIN}/curl" <<'MOCK' +#!/usr/bin/env bash +exit 1 +MOCK +chmod +x "${MOCK_BIN}/curl" + +export STORE_POD_FQDNS=store-0.store-headless +unset HG_SERVER_DRY_RUN || true set +e ( cd "$WORK_ROOT" @@ -44,7 +46,6 @@ set +e ) >"${WORK_ROOT}/out" 2>"${WORK_ROOT}/err" rc=$? set -e - [[ "$rc" -ne 0 ]] || fail "start-server.sh continued after Store health never succeeded" if grep -q 'should not run' "${WORK_ROOT}/out" "${WORK_ROOT}/err"; then fail "start-server.sh reached entrypoint after Store health failed" @@ -52,4 +53,47 @@ fi grep -q 'Store is not healthy' "${WORK_ROOT}/err" \ || fail "start-server.sh did not report Store health failure" +# 2) Only the first Store is healthy — official 3-store compose waits for all. +cat >"${MOCK_BIN}/curl" <<'MOCK' +#!/usr/bin/env bash +url="" +for arg in "$@"; do + case "$arg" in + http://*) url=$arg ;; + esac +done +case "$url" in + *store-0.store-headless:8520*) exit 0 ;; + *) exit 1 ;; +esac +MOCK +chmod +x "${MOCK_BIN}/curl" + +export STORE_POD_FQDNS=store-0.store-headless,store-1.store-headless,store-2.store-headless +set +e +( + cd "$WORK_ROOT" + bash "${ADDON_DIR}/scripts/start-server.sh" +) >"${WORK_ROOT}/out-partial" 2>"${WORK_ROOT}/err-partial" +rc=$? +set -e +[[ "$rc" -ne 0 ]] || fail "start-server.sh continued when only the first Store was healthy" +grep -q 'store-1.store-headless:8520' "${WORK_ROOT}/err-partial" \ + || fail "start-server.sh did not report the unhealthy later Store" + +# 3) All Stores healthy — proceed (dry-run, do not exec the image entrypoint). +cat >"${MOCK_BIN}/curl" <<'MOCK' +#!/usr/bin/env bash +exit 0 +MOCK +chmod +x "${MOCK_BIN}/curl" + +export HG_SERVER_DRY_RUN=1 +( + cd "$WORK_ROOT" + bash "${ADDON_DIR}/scripts/start-server.sh" +) >"${WORK_ROOT}/out-ok" 2>"${WORK_ROOT}/err-ok" +grep -qx 'stores_healthy=3' "${WORK_ROOT}/out-ok" \ + || fail "all-Store wait did not report stores_healthy=3: $(cat "${WORK_ROOT}/out-ok" "${WORK_ROOT}/err-ok")" + echo "HugeGraph start-server Store wait tests passed" From ef535ae43b6f6abe7212dc6371f9d85d7876397c Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Wed, 19 Aug 2026 13:04:02 +0800 Subject: [PATCH 12/16] fix(hugegraph): wait for every PD before store start (#3421) Co-authored-by: Amber --- addons/hugegraph/scripts/start-store.sh | 37 +++++++++--- addons/hugegraph/tests/start_store_test.sh | 65 +++++++++++++++++----- 2 files changed, 81 insertions(+), 21 deletions(-) diff --git a/addons/hugegraph/scripts/start-store.sh b/addons/hugegraph/scripts/start-store.sh index 14695ef5b9..808ac56376 100644 --- a/addons/hugegraph/scripts/start-store.sh +++ b/addons/hugegraph/scripts/start-store.sh @@ -33,20 +33,36 @@ done exit 1 } +pds=() +IFS=',' read -ra hosts <<< "${PD_POD_FQDNS}" +for host in "${hosts[@]}"; do + [[ -n "${host}" ]] || continue + pds+=("${host}") +done +[[ "${#pds[@]}" -ge 1 ]] || { + echo "cannot derive PD list from PD_POD_FQDNS=${PD_POD_FQDNS}" >&2 + exit 1 +} + export HG_STORE_PD_ADDRESS="$(append_port "${PD_POD_FQDNS}" 8686)" -first_pd=${PD_POD_FQDNS%%,*} attempts=${HG_STORE_HEALTH_ATTEMPTS:-60} sleep_secs=${HG_STORE_HEALTH_SLEEP:-2} -pd_healthy=0 +# Official 3-PD compose waits for every PD, not only the first. +# One healthy PD is not a Raft majority. +unhealthy="" for _ in $(seq 1 "${attempts}"); do - if curl -fsS "http://${first_pd}:8620/v1/health" >/dev/null; then - pd_healthy=1 - break - fi + unhealthy="" + for pd in "${pds[@]}"; do + if ! curl -fsS "http://${pd}:8620/v1/health" >/dev/null; then + unhealthy=${pd} + break + fi + done + [[ -z "${unhealthy}" ]] && break sleep "${sleep_secs}" done -[[ "${pd_healthy}" -eq 1 ]] || { - echo "PD is not healthy at http://${first_pd}:8620/v1/health after ${attempts} attempt(s)" >&2 +[[ -z "${unhealthy}" ]] || { + echo "PD is not healthy at http://${unhealthy}:8620/v1/health after ${attempts} attempt(s)" >&2 exit 1 } export HG_STORE_GRPC_HOST="${self}" @@ -55,6 +71,11 @@ export HG_STORE_REST_PORT="${HG_STORE_REST_PORT:-8520}" export HG_STORE_RAFT_ADDRESS="${self}:8510" export HG_STORE_DATA_PATH="${HG_STORE_DATA_PATH:-/hugegraph-store/storage}" +if [[ "${HG_STORE_DRY_RUN:-0}" == "1" ]]; then + printf 'pds_healthy=%s\n' "${#pds[@]}" + exit 0 +fi + mkdir -p "${HG_STORE_DATA_PATH}" cd /hugegraph-store exec /usr/bin/dumb-init -- ./docker-entrypoint.sh diff --git a/addons/hugegraph/tests/start_store_test.sh b/addons/hugegraph/tests/start_store_test.sh index fdb66bb8f9..95cbd9d3b9 100755 --- a/addons/hugegraph/tests/start_store_test.sh +++ b/addons/hugegraph/tests/start_store_test.sh @@ -18,12 +18,6 @@ fail() { mkdir -p "$MOCK_BIN" -cat >"${MOCK_BIN}/curl" <<'MOCK' -#!/usr/bin/env bash -exit 1 -MOCK -chmod +x "${MOCK_BIN}/curl" - cat >"${MOCK_BIN}/dumb-init" <<'MOCK' #!/usr/bin/env bash echo "dumb-init should not run" >&2 @@ -31,19 +25,22 @@ exit 99 MOCK chmod +x "${MOCK_BIN}/dumb-init" -mkdir -p "${WORK_ROOT}/hugegraph-store" -printf '#!/usr/bin/env bash\necho entrypoint should not run\nexit 98\n' \ - >"${WORK_ROOT}/hugegraph-store/docker-entrypoint.sh" -chmod +x "${WORK_ROOT}/hugegraph-store/docker-entrypoint.sh" - export PATH="${MOCK_BIN}:/usr/bin:/bin" -export PD_POD_FQDNS=pd-0.pd-headless export STORE_POD_FQDNS=store-0.store-headless export POD_NAME=store-0 export HG_STORE_HEALTH_ATTEMPTS=2 export HG_STORE_HEALTH_SLEEP=0 export HG_STORE_DATA_PATH="${WORK_ROOT}/store-data" +# 1) Every PD down — fail closed. +cat >"${MOCK_BIN}/curl" <<'MOCK' +#!/usr/bin/env bash +exit 1 +MOCK +chmod +x "${MOCK_BIN}/curl" + +export PD_POD_FQDNS=pd-0.pd-headless +unset HG_STORE_DRY_RUN || true set +e ( cd "$WORK_ROOT" @@ -51,7 +48,6 @@ set +e ) >"${WORK_ROOT}/out" 2>"${WORK_ROOT}/err" rc=$? set -e - [[ "$rc" -ne 0 ]] || fail "start-store.sh continued after PD health never succeeded" if grep -q 'should not run' "${WORK_ROOT}/out" "${WORK_ROOT}/err"; then fail "start-store.sh reached entrypoint after PD health failed" @@ -59,4 +55,47 @@ fi grep -q 'PD is not healthy' "${WORK_ROOT}/err" \ || fail "start-store.sh did not report PD health failure" +# 2) Only the first PD is healthy — official 3-PD compose waits for all. +cat >"${MOCK_BIN}/curl" <<'MOCK' +#!/usr/bin/env bash +url="" +for arg in "$@"; do + case "$arg" in + http://*) url=$arg ;; + esac +done +case "$url" in + *pd-0.pd-headless:8620*) exit 0 ;; + *) exit 1 ;; +esac +MOCK +chmod +x "${MOCK_BIN}/curl" + +export PD_POD_FQDNS=pd-0.pd-headless,pd-1.pd-headless,pd-2.pd-headless +set +e +( + cd "$WORK_ROOT" + bash "${ADDON_DIR}/scripts/start-store.sh" +) >"${WORK_ROOT}/out-partial" 2>"${WORK_ROOT}/err-partial" +rc=$? +set -e +[[ "$rc" -ne 0 ]] || fail "start-store.sh continued when only the first PD was healthy" +grep -q 'pd-1.pd-headless:8620' "${WORK_ROOT}/err-partial" \ + || fail "start-store.sh did not report the unhealthy later PD" + +# 3) All PDs healthy — proceed (dry-run, do not exec the image entrypoint). +cat >"${MOCK_BIN}/curl" <<'MOCK' +#!/usr/bin/env bash +exit 0 +MOCK +chmod +x "${MOCK_BIN}/curl" + +export HG_STORE_DRY_RUN=1 +( + cd "$WORK_ROOT" + bash "${ADDON_DIR}/scripts/start-store.sh" +) >"${WORK_ROOT}/out-ok" 2>"${WORK_ROOT}/err-ok" +grep -qx 'pds_healthy=3' "${WORK_ROOT}/out-ok" \ + || fail "all-PD wait did not report pds_healthy=3: $(cat "${WORK_ROOT}/out-ok" "${WORK_ROOT}/err-ok")" + echo "HugeGraph start-store PD wait tests passed" From c986b81be7dcbcbf7801ba390c4fbc72bd9c3d1c Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Wed, 19 Aug 2026 19:09:55 +0800 Subject: [PATCH 13/16] fix(hugegraph): give distributed health waits official start budget (#3422) Co-authored-by: Amber --- addons/hugegraph/scripts/start-server.sh | 3 ++- addons/hugegraph/scripts/start-store.sh | 3 ++- addons/hugegraph/templates/cmpd-server.yaml | 2 +- addons/hugegraph/templates/cmpd-store.yaml | 2 +- addons/hugegraph/tests/contract_test.sh | 10 ++++++++++ 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/addons/hugegraph/scripts/start-server.sh b/addons/hugegraph/scripts/start-server.sh index 3253269cf3..cc44cff184 100644 --- a/addons/hugegraph/scripts/start-server.sh +++ b/addons/hugegraph/scripts/start-server.sh @@ -35,7 +35,8 @@ export HG_SERVER_USE_PD=true export HG_SERVER_INIT_STORE_ENABLED=false export STORE_REST="${first_store}:8520" -attempts=${HG_SERVER_HEALTH_ATTEMPTS:-60} +# Official Store start_period is 120s. Waiting every Store needs more than 120s. +attempts=${HG_SERVER_HEALTH_ATTEMPTS:-90} sleep_secs=${HG_SERVER_HEALTH_SLEEP:-2} # Official 3-store compose waits for every Store, not only the first. # With pd.initial-store-count = store count, one healthy Store is not diff --git a/addons/hugegraph/scripts/start-store.sh b/addons/hugegraph/scripts/start-store.sh index 808ac56376..2392b1ed6a 100644 --- a/addons/hugegraph/scripts/start-store.sh +++ b/addons/hugegraph/scripts/start-store.sh @@ -45,7 +45,8 @@ done } export HG_STORE_PD_ADDRESS="$(append_port "${PD_POD_FQDNS}" 8686)" -attempts=${HG_STORE_HEALTH_ATTEMPTS:-60} +# Official PD start_period is 120s. 3-PD Raft can exceed a 120s wait. +attempts=${HG_STORE_HEALTH_ATTEMPTS:-90} sleep_secs=${HG_STORE_HEALTH_SLEEP:-2} # Official 3-PD compose waits for every PD, not only the first. # One healthy PD is not a Raft majority. diff --git a/addons/hugegraph/templates/cmpd-server.yaml b/addons/hugegraph/templates/cmpd-server.yaml index 150ce25ec8..cb1a90ed5b 100644 --- a/addons/hugegraph/templates/cmpd-server.yaml +++ b/addons/hugegraph/templates/cmpd-server.yaml @@ -61,7 +61,7 @@ spec: httpGet: path: /versions port: http - failureThreshold: 36 + failureThreshold: 72 periodSeconds: 5 timeoutSeconds: 3 readinessProbe: diff --git a/addons/hugegraph/templates/cmpd-store.yaml b/addons/hugegraph/templates/cmpd-store.yaml index abce0aa5be..6f1a3c28d6 100644 --- a/addons/hugegraph/templates/cmpd-store.yaml +++ b/addons/hugegraph/templates/cmpd-store.yaml @@ -76,7 +76,7 @@ spec: httpGet: path: /v1/health port: rest - failureThreshold: 48 + failureThreshold: 72 periodSeconds: 5 timeoutSeconds: 3 readinessProbe: diff --git a/addons/hugegraph/tests/contract_test.sh b/addons/hugegraph/tests/contract_test.sh index a79211fecd..807b4b2890 100644 --- a/addons/hugegraph/tests/contract_test.sh +++ b/addons/hugegraph/tests/contract_test.sh @@ -206,8 +206,18 @@ assert_contains "${ADDON_DIR}/scripts/start-pd.sh" 'HG_PD_INITIAL_STORE_COUNT=' assert_contains "${ADDON_DIR}/scripts/start-pd.sh" 'count_hosts' assert_contains "${ADDON_DIR}/scripts/start-store.sh" 'HG_STORE_PD_ADDRESS' assert_contains "${ADDON_DIR}/scripts/start-store.sh" 'PD is not healthy' +assert_contains "${ADDON_DIR}/scripts/start-store.sh" 'HG_STORE_HEALTH_ATTEMPTS:-90' assert_contains "${ADDON_DIR}/scripts/start-server.sh" 'HG_SERVER_BACKEND=hstore' assert_contains "${ADDON_DIR}/scripts/start-server.sh" 'Store is not healthy' +assert_contains "${ADDON_DIR}/scripts/start-server.sh" 'HG_SERVER_HEALTH_ATTEMPTS:-90' +assert_equal \ + "$(yq ea '[select(.kind == "ComponentDefinition" and .metadata.name == "hugegraph-store-1.0.0") | .spec.runtime.containers[] | select(.name == "store") | .startupProbe.failureThreshold] | .[0]' "${definition_render}" | tr -d '\n')" \ + "72" \ + "store startupProbe covers PD wait plus process start" +assert_equal \ + "$(yq ea '[select(.kind == "ComponentDefinition" and .metadata.name == "hugegraph-server-1.0.0") | .spec.runtime.containers[] | select(.name == "server") | .startupProbe.failureThreshold] | .[0]' "${definition_render}" | tr -d '\n')" \ + "72" \ + "server startupProbe covers Store wait plus process start" assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'topology: distributed' assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'name: pd' assert_contains "${ROOT_DIR}/examples/hugegraph/cluster-distributed.yaml" 'name: store' From f4b75a17f98b7bfceda7d2a9905dfc4e2eeea745 Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Thu, 20 Aug 2026 00:48:43 +0800 Subject: [PATCH 14/16] test(hugegraph): fail closed on empty distributed FQDNs (#3423) Co-authored-by: Amber --- addons/hugegraph/tests/start_pd_test.sh | 37 +++++++++++++++++++++ addons/hugegraph/tests/start_server_test.sh | 27 +++++++++++++++ addons/hugegraph/tests/start_store_test.sh | 28 ++++++++++++++++ 3 files changed, 92 insertions(+) diff --git a/addons/hugegraph/tests/start_pd_test.sh b/addons/hugegraph/tests/start_pd_test.sh index f97c2e6ce7..fb04372af8 100755 --- a/addons/hugegraph/tests/start_pd_test.sh +++ b/addons/hugegraph/tests/start_pd_test.sh @@ -58,4 +58,41 @@ run_start_pd "${WORK_ROOT}/out5" "${WORK_ROOT}/err5" grep -qx 'HG_PD_INITIAL_STORE_COUNT=5' "${WORK_ROOT}/out5" \ || fail "explicit HG_PD_INITIAL_STORE_COUNT=5 was not honored" +# Empty Store FQDNs must fail closed. Count 0 would bootstrap PD as if no stores exist. +unset HG_PD_INITIAL_STORE_COUNT || true +set +e +PD_POD_FQDNS=pd-0.pd-headless \ +STORE_POD_FQDNS= \ +POD_NAME=pd-0 \ +run_start_pd "${WORK_ROOT}/out-empty-store" "${WORK_ROOT}/err-empty-store" +empty_store_rc=$? +set -e +[[ "$empty_store_rc" -ne 0 ]] || fail "start-pd.sh continued with empty STORE_POD_FQDNS" +grep -q 'STORE_POD_FQDNS is required' "${WORK_ROOT}/err-empty-store" \ + || fail "start-pd.sh did not refuse empty STORE_POD_FQDNS" + +# Comma-only Store FQDNs must fail closed after the required-var check. +set +e +PD_POD_FQDNS=pd-0.pd-headless \ +STORE_POD_FQDNS=, \ +POD_NAME=pd-0 \ +run_start_pd "${WORK_ROOT}/out-blank-store" "${WORK_ROOT}/err-blank-store" +blank_store_rc=$? +set -e +[[ "$blank_store_rc" -ne 0 ]] || fail "start-pd.sh continued with comma-only STORE_POD_FQDNS" +grep -q 'cannot derive store count' "${WORK_ROOT}/err-blank-store" \ + || fail "start-pd.sh did not report comma-only STORE_POD_FQDNS" + +# Empty PD FQDNs must fail closed. PD cannot map this pod to a Raft identity. +set +e +PD_POD_FQDNS= \ +STORE_POD_FQDNS=store-0.store-headless \ +POD_NAME=pd-0 \ +run_start_pd "${WORK_ROOT}/out-empty-pd" "${WORK_ROOT}/err-empty-pd" +empty_pd_rc=$? +set -e +[[ "$empty_pd_rc" -ne 0 ]] || fail "start-pd.sh continued with empty PD_POD_FQDNS" +grep -q 'PD_POD_FQDNS is required' "${WORK_ROOT}/err-empty-pd" \ + || fail "start-pd.sh did not refuse empty PD_POD_FQDNS" + echo "HugeGraph start-pd initial store count tests passed" diff --git a/addons/hugegraph/tests/start_server_test.sh b/addons/hugegraph/tests/start_server_test.sh index 566ae7bf3a..c840f68552 100755 --- a/addons/hugegraph/tests/start_server_test.sh +++ b/addons/hugegraph/tests/start_server_test.sh @@ -96,4 +96,31 @@ export HG_SERVER_DRY_RUN=1 grep -qx 'stores_healthy=3' "${WORK_ROOT}/out-ok" \ || fail "all-Store wait did not report stores_healthy=3: $(cat "${WORK_ROOT}/out-ok" "${WORK_ROOT}/err-ok")" +# Empty Store FQDNs must fail closed before any health wait. +export HG_SERVER_DRY_RUN=1 +export PD_POD_FQDNS=pd-0.pd-headless +export STORE_POD_FQDNS= +set +e +( + cd "$WORK_ROOT" + bash "${ADDON_DIR}/scripts/start-server.sh" +) >"${WORK_ROOT}/out-empty-store" 2>"${WORK_ROOT}/err-empty-store" +empty_store_rc=$? +set -e +[[ "$empty_store_rc" -ne 0 ]] || fail "start-server.sh continued with empty STORE_POD_FQDNS" +grep -q 'STORE_POD_FQDNS is required' "${WORK_ROOT}/err-empty-store" \ + || fail "start-server.sh did not refuse empty STORE_POD_FQDNS" + +export STORE_POD_FQDNS=, +set +e +( + cd "$WORK_ROOT" + bash "${ADDON_DIR}/scripts/start-server.sh" +) >"${WORK_ROOT}/out-blank-store" 2>"${WORK_ROOT}/err-blank-store" +blank_store_rc=$? +set -e +[[ "$blank_store_rc" -ne 0 ]] || fail "start-server.sh continued with comma-only STORE_POD_FQDNS" +grep -q 'cannot derive Store list' "${WORK_ROOT}/err-blank-store" \ + || fail "start-server.sh did not report comma-only STORE_POD_FQDNS" + echo "HugeGraph start-server Store wait tests passed" diff --git a/addons/hugegraph/tests/start_store_test.sh b/addons/hugegraph/tests/start_store_test.sh index 95cbd9d3b9..5cf64205f8 100755 --- a/addons/hugegraph/tests/start_store_test.sh +++ b/addons/hugegraph/tests/start_store_test.sh @@ -98,4 +98,32 @@ export HG_STORE_DRY_RUN=1 grep -qx 'pds_healthy=3' "${WORK_ROOT}/out-ok" \ || fail "all-PD wait did not report pds_healthy=3: $(cat "${WORK_ROOT}/out-ok" "${WORK_ROOT}/err-ok")" +# Empty PD FQDNs must fail closed before any health wait. +export HG_STORE_DRY_RUN=1 +export STORE_POD_FQDNS=store-0.store-headless +export POD_NAME=store-0 +export PD_POD_FQDNS= +set +e +( + cd "$WORK_ROOT" + bash "${ADDON_DIR}/scripts/start-store.sh" +) >"${WORK_ROOT}/out-empty-pd" 2>"${WORK_ROOT}/err-empty-pd" +empty_pd_rc=$? +set -e +[[ "$empty_pd_rc" -ne 0 ]] || fail "start-store.sh continued with empty PD_POD_FQDNS" +grep -q 'PD_POD_FQDNS is required' "${WORK_ROOT}/err-empty-pd" \ + || fail "start-store.sh did not refuse empty PD_POD_FQDNS" + +export PD_POD_FQDNS=, +set +e +( + cd "$WORK_ROOT" + bash "${ADDON_DIR}/scripts/start-store.sh" +) >"${WORK_ROOT}/out-blank-pd" 2>"${WORK_ROOT}/err-blank-pd" +blank_pd_rc=$? +set -e +[[ "$blank_pd_rc" -ne 0 ]] || fail "start-store.sh continued with comma-only PD_POD_FQDNS" +grep -q 'cannot derive PD list' "${WORK_ROOT}/err-blank-pd" \ + || fail "start-store.sh did not report comma-only PD_POD_FQDNS" + echo "HugeGraph start-store PD wait tests passed" From f9e598f6c033c39ea1316ac70261d02e1c506685 Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Thu, 20 Aug 2026 03:53:12 +0800 Subject: [PATCH 15/16] fix(hugegraph): expose claimed distributed shape in cluster schema (#3424) Co-authored-by: Amber --- addons-cluster/hugegraph/values.schema.json | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/addons-cluster/hugegraph/values.schema.json b/addons-cluster/hugegraph/values.schema.json index 6a7f9b8c21..12313a48ea 100644 --- a/addons-cluster/hugegraph/values.schema.json +++ b/addons-cluster/hugegraph/values.schema.json @@ -9,6 +9,13 @@ "default": "1.7.0", "enum": ["1.7.0"] }, + "topology": { + "title": "Topology", + "description": "standalone is one Server with RocksDB. distributed is PD 3 + Store 3 + Server 1.", + "type": "string", + "default": "standalone", + "enum": ["standalone", "distributed"] + }, "replicas": { "title": "Replicas", "description": "Standalone HugeGraph requires exactly one replica.", @@ -17,6 +24,34 @@ "minimum": 1, "maximum": 1 }, + "distributed": { + "title": "Distributed", + "description": "Claimed distributed shape. Horizontal scaling is not supported.", + "type": "object", + "properties": { + "pdReplicas": { + "title": "PD Replicas", + "type": "integer", + "default": 3, + "minimum": 3, + "maximum": 3 + }, + "storeReplicas": { + "title": "Store Replicas", + "type": "integer", + "default": 3, + "minimum": 3, + "maximum": 3 + }, + "serverReplicas": { + "title": "Server Replicas", + "type": "integer", + "default": 1, + "minimum": 1, + "maximum": 1 + } + } + }, "cpu": { "title": "CPU", "description": "CPU cores.", From d5e2d25f1f9a39053f235d2a9307bd9dd2df2cdd Mon Sep 17 00:00:00 2001 From: Wei Cao Date: Thu, 20 Aug 2026 05:24:09 +0800 Subject: [PATCH 16/16] test(hugegraph): pin cluster schema claimed distributed shape (#3425) Co-authored-by: Amber --- addons/hugegraph/tests/contract_test.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/addons/hugegraph/tests/contract_test.sh b/addons/hugegraph/tests/contract_test.sh index 807b4b2890..a21ab80930 100644 --- a/addons/hugegraph/tests/contract_test.sh +++ b/addons/hugegraph/tests/contract_test.sh @@ -246,6 +246,16 @@ assert_equal \ "pd,store,server" \ "distributed component names" rm -f "${distributed_render}" +schema="${CLUSTER_DIR}/values.schema.json" +assert_file "${schema}" +assert_equal \ + "$(yq -oy '.properties.topology.enum | join(",")' "${schema}")" \ + "standalone,distributed" \ + "cluster schema topology enum" +assert_equal \ + "$(yq -oy '[.properties.distributed.properties.pdReplicas.default, .properties.distributed.properties.storeReplicas.default, .properties.distributed.properties.serverReplicas.default] | join("/")' "${schema}")" \ + "3/3/1" \ + "cluster schema claimed distributed defaults" assert_contains "${ADDON_DIR}/.helmignore" '^exporter/' package_dir=$(mktemp -d)