diff --git a/.codecov.yml b/.codecov.yml index cd52e2604d4..009acfb0288 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -36,3 +36,13 @@ ignore: - "src/tests/" - "include/xrpl/beast/test/" - "include/xrpl/beast/unit_test/" + # Telemetry modules — conditionally compiled behind XRPL_ENABLE_TELEMETRY, + # which is not enabled in coverage builds. + - "src/xrpld/telemetry/" + - "src/libxrpl/telemetry/" + - "include/xrpl/telemetry/" + - "src/libxrpl/beast/insight/OTelCollector.cpp" + - "include/xrpl/beast/insight/OTelCollector.h" + # Per-module span-name constant headers (compile-time constants only, + # colocated with their subsystem rather than under telemetry/). + - "**/*SpanNames.h" diff --git a/.cspell.config.yaml b/.cspell.config.yaml index e220cd02491..981f8e570e8 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -65,6 +65,7 @@ words: - Btrfs - Buildx - canonicality + - CGNAT - canonicalised - changespq - checkme @@ -98,6 +99,8 @@ words: - dearmor - decryptor - dedented + - Dedup + - dedup - deleteme - demultiplexer - deserializaton @@ -107,14 +110,17 @@ words: - disablerepo - distro - doxyfile + - dthms - dxrpl - elgamal - enabled - enablerepo - endmacro - envrc + - EOCFG - exceptioned - EXPECT_STREQ + - exfiltration - Falco - fcontext - finalizers @@ -123,6 +129,8 @@ words: - fsanitize - funclets - Gamal + - gantt + - Gantt - gcov - gcovr - ghead @@ -130,11 +138,15 @@ words: - godexsoft - gpgcheck - gpgkey + - havetxset - hotwallet + - hicpp + - htpasswd - hwaddress - hwrap - ifndef - inequation + - initialiser - insuf - insuff - invasively @@ -162,6 +174,7 @@ words: - libxrpl - llection - LOCALGOOD + - logql - logwstream - lseq - lsmf @@ -169,12 +182,11 @@ words: - mathbunnyru - mcmodel - MEMORYSTATUSEX - - MPTAMM - - MPTDEX - Merkle - Metafuncton - misprediction - missingok + - MPTAMM - mptbalance - MPTDEX - mptflags @@ -203,11 +215,13 @@ words: - nixfmt - nixos - nixpkgs + - NETOP - NOLINT - NOLINTNEXTLINE - nonxrp - noreplace - noripple + - nostd - nostdinc - notifempty - nudb @@ -216,6 +230,8 @@ words: - Nyffenegger - onlatest - ostr + - otelc + - otelcol - pargs - partitioner - paychan @@ -224,8 +240,12 @@ words: - permdex - perminute - permissioned + - pgrep + - pkill + - pimpl - pointee - populator + - pratik - preauth - preauthorization - preauthorize @@ -243,7 +263,9 @@ words: - queuable - Raphson - rcflags + - reparent - replayer + - reqps - rerandomize - rerandomization - rerandomized @@ -265,6 +287,11 @@ words: - rustfmt - rustup - sahyadri + - sanitisation + - sanitise + - sanitised + - sanitiser + - sanitising - Satoshi - scons - Schnorr @@ -284,6 +311,7 @@ words: - sles - soci - socidb + - speciality - sponsee - sponsees - SRPMS @@ -291,6 +319,7 @@ words: - statsd - STATSDCOLLECTOR - stissue + - stledgerentry - stnum - stobj - stobject @@ -309,6 +338,7 @@ words: - TMEndpointv2 - toolchain - tparam + - traceql - trixie - tx - txid @@ -316,6 +346,7 @@ words: - txjson - txn - txns + - txqueue - txs - ubsan - UBSAN @@ -340,6 +371,7 @@ words: - unsquelch - unsquelched - unsquelching + - unstored - unvalidated - unveto - unvetoed @@ -350,6 +382,7 @@ words: - vfalco - vinnie - wasmi + - werror - wextra - wptr - writeme @@ -365,4 +398,7 @@ words: - xrplf - xxhash - xxhasher - - CGNAT + - xychart + - zpages + - ripplex + - mseconds diff --git a/.github/scripts/levelization/results/loops.txt b/.github/scripts/levelization/results/loops.txt index cf70468e32c..0641538e6d9 100644 --- a/.github/scripts/levelization/results/loops.txt +++ b/.github/scripts/levelization/results/loops.txt @@ -10,6 +10,12 @@ Loop: xrpld.app xrpld.rpc Loop: xrpld.app xrpld.shamap xrpld.shamap > xrpld.app +Loop: xrpld.app xrpld.telemetry + xrpld.app > xrpld.telemetry + Loop: xrpld.overlay xrpld.rpc xrpld.rpc ~= xrpld.overlay +Loop: xrpld.overlay xrpld.telemetry + xrpld.telemetry ~= xrpld.overlay + diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index 3c9c5145165..ab039431ca3 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -48,6 +48,10 @@ libxrpl.shamap > xrpl.basics libxrpl.shamap > xrpl.nodestore libxrpl.shamap > xrpl.protocol libxrpl.shamap > xrpl.shamap +libxrpl.telemetry > xrpl.basics +libxrpl.telemetry > xrpl.config +libxrpl.telemetry > xrpl.protocol +libxrpl.telemetry > xrpl.telemetry libxrpl.tx > xrpl.basics libxrpl.tx > xrpl.conditions libxrpl.tx > xrpl.core @@ -55,6 +59,7 @@ libxrpl.tx > xrpl.json libxrpl.tx > xrpl.ledger libxrpl.tx > xrpl.protocol libxrpl.tx > xrpl.server +libxrpl.tx > xrpl.telemetry libxrpl.tx > xrpl.tx test.app > test.jtx test.app > test.unit_test @@ -107,6 +112,7 @@ test.csf > xrpl.basics test.csf > xrpld.consensus test.csf > xrpl.json test.csf > xrpl.ledger +test.csf > xrpl.telemetry test.json > test.jtx test.json > xrpl.json test.jtx > test.unit_test @@ -193,6 +199,7 @@ test.unit_test > xrpl.protocol tests.libxrpl > xrpl.basics tests.libxrpl > xrpl.config tests.libxrpl > xrpl.core +tests.libxrpl > xrpld.telemetry tests.libxrpl > xrpl.json tests.libxrpl > xrpl.ledger tests.libxrpl > xrpl.net @@ -202,6 +209,7 @@ tests.libxrpl > xrpl.protocol_autogen tests.libxrpl > xrpl.resource tests.libxrpl > xrpl.server tests.libxrpl > xrpl.shamap +tests.libxrpl > xrpl.telemetry tests.libxrpl > xrpl.tx xrpl.conditions > xrpl.basics xrpl.conditions > xrpl.protocol @@ -239,10 +247,13 @@ xrpl.server > xrpl.resource xrpl.shamap > xrpl.basics xrpl.shamap > xrpl.nodestore xrpl.shamap > xrpl.protocol +xrpl.telemetry > xrpl.basics +xrpl.telemetry > xrpl.config xrpl.tx > xrpl.basics xrpl.tx > xrpl.core xrpl.tx > xrpl.ledger xrpl.tx > xrpl.protocol +xrpl.tx > xrpl.telemetry xrpld.app > test.unit_test xrpld.app > xrpl.basics xrpld.app > xrpl.config @@ -258,11 +269,13 @@ xrpld.app > xrpl.rdb xrpld.app > xrpl.resource xrpld.app > xrpl.server xrpld.app > xrpl.shamap +xrpld.app > xrpl.telemetry xrpld.app > xrpl.tx xrpld.consensus > xrpl.basics xrpld.consensus > xrpl.json xrpld.consensus > xrpl.ledger xrpld.consensus > xrpl.protocol +xrpld.consensus > xrpl.telemetry xrpld.core > xrpl.basics xrpld.core > xrpl.config xrpld.core > xrpl.core @@ -281,6 +294,7 @@ xrpld.overlay > xrpl.protocol xrpld.overlay > xrpl.resource xrpld.overlay > xrpl.server xrpld.overlay > xrpl.shamap +xrpld.overlay > xrpl.telemetry xrpld.overlay > xrpl.tx xrpld.peerfinder > xrpl.basics xrpld.peerfinder > xrpl.config @@ -292,6 +306,7 @@ xrpld.perflog > xrpl.config xrpld.perflog > xrpl.core xrpld.perflog > xrpld.app xrpld.perflog > xrpld.rpc +xrpld.perflog > xrpld.telemetry xrpld.perflog > xrpl.json xrpld.perflog > xrpl.nodestore xrpld.perflog > xrpl.protocol @@ -308,9 +323,20 @@ xrpld.rpc > xrpl.rdb xrpld.rpc > xrpl.resource xrpld.rpc > xrpl.server xrpld.rpc > xrpl.shamap +xrpld.rpc > xrpl.telemetry xrpld.rpc > xrpl.tx xrpld.shamap > xrpl.basics xrpld.shamap > xrpld.core xrpld.shamap > xrpl.nodestore xrpld.shamap > xrpl.protocol xrpld.shamap > xrpl.shamap +xrpld.telemetry > xrpl.basics +xrpld.telemetry > xrpl.core +xrpld.telemetry > xrpld.consensus +xrpld.telemetry > xrpld.core +xrpld.telemetry > xrpl.json +xrpld.telemetry > xrpl.nodestore +xrpld.telemetry > xrpl.protocol +xrpld.telemetry > xrpl.rdb +xrpld.telemetry > xrpl.server +xrpld.telemetry > xrpl.telemetry diff --git a/.github/scripts/otel-naming/README.md b/.github/scripts/otel-naming/README.md new file mode 100644 index 00000000000..660b574ed06 --- /dev/null +++ b/.github/scripts/otel-naming/README.md @@ -0,0 +1,70 @@ +# OTel naming-consistency check + +`check_otel_naming.py` enforces the OpenTelemetry span-attribute naming +convention documented in +[CONTRIBUTING.md](../../../CONTRIBUTING.md#telemetry-span-attribute-naming) +across every layer of the telemetry pipeline. The `*SpanNames.h` constants are +the single source of truth (L1); every other layer must agree with them. + +## Running locally + +``` +python .github/scripts/otel-naming/check_otel_naming.py +``` + +It takes no arguments, can be run from any directory inside the repo, and uses +only the Python standard library (no `pip install`, matching the levelization +check). A non-zero exit code means a violation was found; the output lists each +violation as `RULE | location | token | expected`. + +## What it checks + +The valid key set is **derived dynamically from the OTel code** — there is no +hardcoded allowlist: + +- **L1 keys** come from the `namespace attr { ... }` blocks of every + `*SpanNames.h`, resolving the `makeStr("x")` / `join(seg::a, seg::b)` DSL + (cross-file, so `join(seg::rpc, ...)` resolves `seg::rpc` from the base + `SpanNames.h`). Each constant is resolved against **its own** header, so two + headers that define a same-named constant (e.g. a base `attr::ledgerHash` and + a domain `attr::ledgerHash`) each contribute their real wire key — a later + header cannot clobber an earlier one's value in a flat table. +- **Legitimate dotted keys** = ONLY the keys the code actually sets as resource + attributes, i.e. the entries inside `Telemetry.cpp`'s `Resource::Create({...})` + call: the `semconv::service::*` keys (`service.*`) plus any `attr::` + constants passed there (`xrpl.network.*`). A dotted key that is _declared_ in a + header but never set as a resource attr is a span attribute in resource + clothing — a Rule-A violation, even if it lives in the base `SpanNames.h`. + +### Rules (each fails the build, when its inputs are present) + +| Rule | Check | +| ---- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| A | No stray dotted span-attribute key (only the derived resource keys may be dotted). | +| G | Attribute keys are `lower_snake_case` (`^[a-z][a-z0-9_]*$` per dot-segment) — no camelCase, UPPERCASE, or spaces. | +| F | No string literals as attribute keys or span-name arguments in `setAttribute`/`addEvent`/`span`/`rootSpan`/`childSpan` (`rootSpan` shares `span`'s `(cat, prefix, name)` signature). Attribute _values_ are exempt (runtime data); `*SpanNames.h` definitions and test files are exempt. | +| B | Every collector `spanmetrics.dimensions` name exists in the L1 key set. | +| C | Every Tempo span-filter tag exists in the L1 key set. | +| D | Every dashboard label resolves to an L1 span attribute, a native-metric label (L6, emitted by MetricsRegistry), or a Prometheus/Grafana builtin. TraceQL scope prefixes (`span.`/`resource.`/…) are stripped before the L1 lookup. | +| E | No dotted `xrpl..` attribute key in the runbook (only the L1 resource attrs `xrpl.network.*` may be dotted). Span names, filenames, OTel-standard keys, and metric labels are not flagged. | + +Rule F runs **unconditionally** (it is a purely syntactic check on the +call-sites and needs no `*SpanNames.h`), so a code path that calls +`SpanGuard::span`/`setAttribute` directly without ever defining a header is +still caught. + +### Warnings (printed, never fail the build) + +| Rule | Check | +| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| H | A namespace-qualified constant (e.g. `foo::bar::myKey`) used at a telemetry call-site is not defined in any `*SpanNames.h`. The constant should live in the proper header; defining it in-place bypasses rules A/G/F. Warns rather than fails — the argument may be a legitimately dynamic value, and the header may live on a later branch. Bare locals and `std::` names are not warned. | + +## Presence-gated + +Every rule runs **only when the source files it needs are present** in the tree +and is otherwise skipped (printed as `SKIP: — `), never failed. +This keeps the check correct no matter how telemetry work is split across PRs — +a stacked chain, one large PR, or independent per-stage PRs where (for example) +the collector config lands before the dashboards. The collector/Tempo/dashboard/ +runbook layers are introduced in later phases; on a branch without them, only +the L1-intrinsic rules (A, G, F) run. diff --git a/.github/scripts/otel-naming/check_otel_naming.py b/.github/scripts/otel-naming/check_otel_naming.py new file mode 100644 index 00000000000..89d02d3339f --- /dev/null +++ b/.github/scripts/otel-naming/check_otel_naming.py @@ -0,0 +1,932 @@ +#!/usr/bin/env python3 + +""" +Usage: check_otel_naming.py +This script takes no parameters and can be called from any directory inside the +repository (it locates the repo root via `git rev-parse`). + +Enforces the OpenTelemetry span-attribute naming convention documented in +CONTRIBUTING.md ("Telemetry span attribute naming") across every layer of the +telemetry pipeline. The `*SpanNames.h` constants are the single source of truth +(L1); every other layer must agree with them. + +Design principles +----------------- +1. No hardcoded allowlist. The set of valid attribute keys — including which + dotted keys are legitimate resource attributes — is derived dynamically by + parsing the repository's own OTel code: + * `*SpanNames.h` `namespace attr { ... }` blocks (the underscore/bare keys + and the `join(seg::..., ...)` dotted resource compositions), and + * the keys the code passes to `Resource::Create({ ... })` in Telemetry.cpp + (the standard `semconv::service::*` keys -> service.name/version/...). + The one narrow, explicit exception is EXTERNAL_INFRA_LABELS (Rules D & E): + identity labels stamped by infrastructure outside this repo's OTel code + (the perf-iac harness), which by definition have no source in-tree to + derive from. Kept separate from the generic Prometheus/Grafana builtins + set so the exception stays visible rather than blending into "things + every OTel setup has". perf-iac's alloy pipeline stamps each identity at + two layers -- dotted on the OTel resource attribute (xrpl.work.item/ + .branch/.node.role, checked by Rule E) and underscore on the derived + Prometheus metric-datapoint label (xrpl_work_item/_branch/_node_role, + checked by Rule D) -- so both forms are exempt from the same constant. + +2. Presence-gated enforcement. Every rule runs ONLY when the source files it + needs are present in the tree, and is otherwise skipped (never failed). This + keeps the check correct no matter how work is split across PRs: a stacked + chain, one large PR, or independent per-stage PRs where (for example) the + collector config lands in a different PR than the dashboards. The check never + assumes a file from another phase/PR exists. + +Layers +------ + L1 code : src/**/*SpanNames.h, include/**/*SpanNames.h (ground truth) + L1 resource : src/libxrpl/telemetry/Telemetry.cpp (dotted allowlist) + L1 callsites : setAttribute/addEvent/span/rootSpan/childSpan in src/**, + include/** + L2 collector : docker/telemetry/otel-collector-config.yaml (spanmetrics dims) + L3 tempo : docker/telemetry/tempo.yaml (span filter tags) + L4 dashboards: docker/telemetry/grafana/dashboards/*.json (PromQL labels) + L5 runbook : docs/telemetry-runbook.md (attr tables) + L6 metrics : MetricsRegistry.cpp instrument labels (native-metric + label keys, a valid dashboard-label source besides L1) + +Rules (each FAILS the build, when its inputs are present) +--------------------------------------------------------- + A No stray dotted span-attribute key. A dotted `.` used as a span + attribute that is not in the derived resource-key set is a violation. + G Attribute keys must be lower_snake_case (^[a-z][a-z0-9_]*$ per segment). + Flags camelCase, UPPERCASE, spaces, and other stray characters. + F No string literals as attribute keys or span-name arguments. The + setAttribute/addEvent key and the span/rootSpan/childSpan prefix/name args + must reference a *SpanNames.h constant, never a "literal". Attribute VALUES + are exempt (runtime data). Definitions inside *SpanNames.h are exempt, and + test files are exempt (they pass arbitrary literals to exercise the API). + B Every collector spanmetrics dimension exists in the L1 key set. + C Every tempo span-filter tag exists in the L1 key set. + D Every dashboard label resolves to an L1 span attribute, an L6 + native-metric label, or a builtin. TraceQL `span.`/`resource.` scope + prefixes are stripped before the L1 lookup. + E No dotted `xrpl..` attribute key in the runbook (only the + L1 resource attrs xrpl.network.* and the EXTERNAL_INFRA_LABELS dotted + form -- xrpl.work.item/.branch/.node.role -- may be dotted). Span names, + filenames, + OTel-standard keys, and metric labels are not flagged. + +Warnings (printed, but do NOT fail the build) +---------------------------------------------- + H A constant referenced at a telemetry call-site is not defined in any + *SpanNames.h. Span constants should live in the corresponding + *SpanNames.h (single source of truth); defining one in-place bypasses the + naming rules. A warning (not a failure) because the argument may instead + be a legitimately dynamic local (e.g. a computed span-name leaf). + +Exit code is non-zero if any present-and-enforced rule finds a violation. +Warnings never change the exit code. +""" + +import re +import subprocess +import sys +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple + +# --------------------------------------------------------------------------- +# Repo location +# --------------------------------------------------------------------------- + + +def repo_root() -> Path: + """Return the repository root, so the script works from any CWD. + + Exits with a readable message (not a traceback) if git is unavailable or the + CWD is outside a repository.""" + try: + out = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=True, + ) + except (subprocess.CalledProcessError, FileNotFoundError): + print( + "error: check_otel_naming.py must be run inside the git repository.", + file=sys.stderr, + ) + sys.exit(2) + return Path(out.stdout.strip()) + + +def read_source(path: Path) -> str: + """Read a file as UTF-8, tolerating stray non-UTF-8 bytes rather than + crashing the whole check on one bad byte.""" + return path.read_text(encoding="utf-8", errors="ignore") + + +# --------------------------------------------------------------------------- +# Regexes (compiled once) +# --------------------------------------------------------------------------- + +# A segment/string constant definition: `inline constexpr auto NAME = ;` +CONST_DEF = re.compile(r"inline\s+constexpr\s+auto\s+(\w+)\s*=\s*(.+?);", re.DOTALL) +MAKESTR = re.compile(r'makeStr\(\s*"([^"]*)"\s*\)') +# A `namespace {` opener, to track which namespace a constant lives in. +NS_OPEN = re.compile(r"namespace\s+([\w:]+)\s*\{") +# A `using ::a::b::field;` re-export inside an attr block; captures the leaf. +USING_DECL = re.compile(r"using\s+(?:::)?[\w:]*::(\w+)\s*;") +# Telemetry call-sites whose string arguments must be constants, not literals. +# Require a receiver so we match real SpanGuard calls, not std::span / a math +# `span(...)` / a bare method declaration: +# - `SpanGuard::span(` / `SpanGuard::rootSpan(` / `SpanGuard::childSpan(` +# (static factories) +# - `.span(` / `->setAttribute(` etc. (member call) +# `span`/`rootSpan`/`childSpan` additionally require the `SpanGuard`/`.`/`->` +# receiver; `setAttribute`/`addEvent` only ever exist on a guard, so a `.`/`->` +# suffices. `rootSpan` shares `span`'s (cat, prefix, name) signature. +CALLSITE = re.compile( + r"(?:SpanGuard::|\.|->)\s*(setAttribute|addEvent|span|rootSpan|childSpan)\s*\(" +) +# A C++ string literal (used to flag literals inside call-site argument lists). +STRING_LITERAL = re.compile(r'"((?:[^"\\]|\\.)*)"') +# A C++ line comment (`//` ... end of line) and a block comment (`/* ... */`). +LINE_COMMENT = re.compile(r"//[^\n]*") +BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL) +# A TraceQL scope prefix on a label (`span.`, `resource.`, `event.`, etc.). +# Dashboards reference span attributes in TraceQL as `span.`; the bare +# attribute is what must exist in L1, so strip the scope before validating. +TRACEQL_SCOPE = re.compile(r"^(?:span|resource|event|link|instrumentation_scope)\.") +# An OTel metric label key as emitted in C++: `Add(.., {{"label", ...}})` / +# `{{"label", value}}` instrument calls in MetricsRegistry. +METRIC_LABEL = re.compile(r'\{\{\s*"([a-z_][a-z0-9_]*)"\s*,') + + +def strip_comments(text: str) -> str: + """Remove C/C++ `//` line comments and `/* ... */` block comments. + + Used only for L1 attribute-key extraction so that a commented-out or + illustrative `makeStr("...")` inside a `namespace attr` block does not leak + into the authoritative key set. Rule F deliberately does NOT strip comments + — it must still see `@code` doc-comment examples so their call-site + arguments are held to the constant-only convention. + + String literals are not specially handled; a `//` or `/*` appearing inside a + string is vanishingly rare in the *SpanNames.h headers and would at worst + drop a constant from L1 (a conservative direction). + """ + text = BLOCK_COMMENT.sub("", text) + text = LINE_COMMENT.sub("", text) + return text + + +# --------------------------------------------------------------------------- +# L1: parse *SpanNames.h into the authoritative key set +# --------------------------------------------------------------------------- + + +def find_spanname_headers(root: Path) -> List[Path]: + return sorted( + p + for p in list((root / "src").rglob("*SpanNames.h")) + + list((root / "include").rglob("*SpanNames.h")) + if p.is_file() + ) + + +def resolve_constants( + text: str, symbols: Optional[Dict[str, str]] = None +) -> Dict[str, str]: + """Resolve `inline constexpr auto NAME = ` to strings. + + Supports the small constexpr DSL used by SpanNames.h: + makeStr("x") -> "x" + join(a, b) -> resolve(a) + "." + resolve(b) + seg::xrpl / attr::foo -> looked up in the symbol table + The optional `symbols` argument seeds (and is updated in place with) the + table, so a global pass over ALL *SpanNames.h headers can resolve + cross-file references such as `join(seg::rpc, ...)` where `seg::rpc` is + defined in the base SpanNames.h. Keys are stored by their bare name + (last `::` component), so `seg::rpc` and `rpc` both resolve. + """ + if symbols is None: + symbols = {} + + def resolve_expr(expr: str) -> Optional[str]: + expr = expr.strip() + m = MAKESTR.fullmatch(expr) + if m: + return m.group(1) + if expr.startswith("join(") and expr.endswith(")"): + args = split_top_level_args(expr[len("join(") : -1]) + parts = [resolve_expr(a) for a in args] + if any(p is None for p in parts): + return None + return ".".join(p for p in parts if p is not None) + # Bare or qualified symbol reference, e.g. `seg::xrpl` or `networkId`. + key = expr.split("::")[-1] + return symbols.get(key, symbols.get(expr)) + + # Iterate definitions in source order so earlier symbols are available. + for m in CONST_DEF.finditer(text): + name, expr = m.group(1), m.group(2) + val = resolve_expr(expr) + if val is not None: + symbols[name] = val + return symbols + + +def build_global_symbols(headers: List[Path]) -> Dict[str, str]: + """Resolve constants across ALL headers so cross-file `seg::`/`join` + references (e.g. `join(seg::rpc, ...)` in RpcSpanNames.h, where `seg::rpc` + lives in the base SpanNames.h) resolve. Base SpanNames.h is processed + first so its `seg::` segments seed the table.""" + symbols: Dict[str, str] = {} + ordered = sorted(headers, key=lambda p: (p.name != "SpanNames.h", str(p))) + # Two passes: the first seeds segments, the second resolves dependents. + # Comments are stripped so a commented-out constant cannot seed the table. + for _ in range(2): + for h in ordered: + resolve_constants(strip_comments(read_source(h)), symbols) + return symbols + + +def split_top_level_args(s: str) -> List[str]: + """Split a comma-separated arg list, respecting nested parentheses and + ignoring parens/commas that appear inside a "string literal" (so a value + like `setAttribute(k, ",")` does not get mis-split).""" + args, depth, cur = [], 0, "" + in_str = False + escaped = False + for ch in s: + if in_str: + cur += ch + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == '"': + in_str = False + continue + if ch == '"': + in_str = True + cur += ch + elif ch == "(": + depth += 1 + cur += ch + elif ch == ")": + depth -= 1 + cur += ch + elif ch == "," and depth == 0: + args.append(cur) + cur = "" + else: + cur += ch + if cur.strip(): + args.append(cur) + return args + + +def attr_namespace_spans(text: str) -> List[str]: + """Return the source text of each `namespace attr { ... }` block in `text`. + + Brace-matched over the whole (comment-stripped) text, so a definition that + wraps across several physical lines is contained in one span. Nested braces + inside the block are balanced correctly.""" + spans: List[str] = [] + for opener in NS_OPEN.finditer(text): + if opener.group(1).split("::")[-1] != "attr": + continue + # Walk from the opening brace, balancing nesting to the matching close. + i = opener.end() # one char past the namespace's `{` + depth = 1 + start = i + while i < len(text) and depth > 0: + c = text[i] + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + i += 1 + spans.append(text[start : i - 1]) + return spans + + +def attr_keys_from_header(path: Path, symbols: Dict[str, str]) -> Set[str]: + """Return the set of attribute-key strings declared in a header's + `namespace attr { ... }` block(s). `symbols` is the global cross-file + table, used ONLY to seed `seg::`/segment references for `join(...)` + resolution — never to look up an attr constant's value. + + A constant DEFINED in this header is resolved against this header's OWN + text, so two headers that each define a same-named constant (e.g. the base + `attr::ledgerHash = xrpl.ledger.hash` and consensus + `attr::ledgerHash = ledger_hash`) each report their real wire key. The + global table is keyed by bare name and would otherwise let a later header + clobber an earlier one, erasing the real key from L1 (a Rule-A blind spot). + A `using`-re-export, by contrast, imports a constant defined elsewhere, so + it is resolved against the global table. + + Comments are stripped first (a commented constant must not enter L1), and + each attr block is brace-matched over the whole text so multi-line + `inline constexpr auto NAME = join(\\n ...);` definitions are captured.""" + text = strip_comments(read_source(path)) + # Local table: the global segments/symbols seed cross-file `join` parts, + # then this header's own definitions overwrite any same-named global entry + # so a locally-defined attr resolves to ITS value, not another header's. + local = dict(symbols) + resolve_constants(text, local) + keys: Set[str] = set() + for block in attr_namespace_spans(text): + for md in CONST_DEF.finditer(block): + # Resolve a locally-defined constant against the LOCAL table; this + # captures makeStr("x") and join(seg::y, ...) with the header's own + # value, immune to cross-header bare-name collisions. + val = local.get(md.group(1)) + if val is not None: + keys.add(val) + # `using ::ns::attr::field;` re-exports a constant defined in ANOTHER + # header (e.g. PeerSpanNames imports the base ledgerHash). Resolve the + # imported name against the global table. + for um in USING_DECL.finditer(block): + val = symbols.get(um.group(1)) + if val is not None: + keys.add(val) + return keys + + +# --------------------------------------------------------------------------- +# Reporting +# --------------------------------------------------------------------------- + + +class Report: + def __init__(self) -> None: + self.violations: List[Tuple[str, str, str, str]] = [] + self.warnings: List[Tuple[str, str, str, str]] = [] + self.skips: List[str] = [] + self.checked: List[str] = [] + + def violation(self, rule: str, loc: str, token: str, expected: str) -> None: + self.violations.append((rule, loc, token, expected)) + + def warning(self, rule: str, loc: str, token: str, note: str) -> None: + """A non-fatal finding: printed, but does not fail the build. Used where + the script cannot be certain a finding is wrong (e.g. a constant used at + a call-site that is not defined in any *SpanNames.h — it might be a + misplaced constant, or a legitimately dynamic value).""" + self.warnings.append((rule, loc, token, note)) + + def skip(self, rule: str, reason: str) -> None: + self.skips.append(f"SKIP: {rule} — {reason}") + + def ok(self, msg: str) -> None: + self.checked.append(f"OK: {msg}") + + def render_and_exit(self) -> None: + for line in self.skips: + print(line) + for line in self.checked: + print(line) + if self.warnings: + print("\nNaming-convention warnings (non-fatal):\n") + print(f" {'RULE':<5} {'LOCATION':<48} {'TOKEN':<28} NOTE") + print(f" {'-' * 5} {'-' * 48} {'-' * 28} {'-' * 30}") + for rule, loc, token, note in self.warnings: + print(f" {rule:<5} {loc:<48} {token:<28} {note}") + if self.violations: + print("\nNaming-convention violations:\n") + print(f" {'RULE':<5} {'LOCATION':<48} {'TOKEN':<28} EXPECTED") + print(f" {'-' * 5} {'-' * 48} {'-' * 28} {'-' * 30}") + for rule, loc, token, expected in self.violations: + print(f" {rule:<5} {loc:<48} {token:<28} {expected}") + print( + "\nSee CONTRIBUTING.md -> 'Telemetry span attribute naming'. " + "The *SpanNames.h constants are the single source of truth." + ) + sys.exit(1) + print("\nAll present telemetry naming layers are consistent.") + sys.exit(0) + + +def main() -> None: + root = repo_root() + report = Report() + + # --- Build the L1 ground-truth key set (presence-gated) ---------------- + headers = find_spanname_headers(root) + l1_keys: Set[str] = set() + if headers: + symbols = build_global_symbols(headers) + # Map each key to the header(s) that declare it, so Rule A can tell a + # legitimate resource attr (declared in the base SpanNames.h) from a + # stray dotted key declared in a domain header. + keys_by_header: Dict[Path, Set[str]] = {} + for h in headers: + hk = attr_keys_from_header(h, symbols) + keys_by_header[h] = hk + l1_keys |= hk + report.ok( + f"L1: {len(l1_keys)} attribute keys from {len(headers)} " + f"*SpanNames.h header(s)" + ) + else: + report.skip("L1", "no *SpanNames.h present (not a naming-relevant tree)") + keys_by_header = {} + + # --- Derive the legitimate dotted (resource) keys dynamically ---------- + # ONLY the keys actually passed to Resource::Create() in Telemetry.cpp + # (semconv service.* + the attr:: constants set there, e.g. xrpl.network.*). + # A dotted key declared in a header but NOT set as a resource attr is a + # Rule-A violation, not an allowlist entry. + resource_symbols = symbols if headers else {} + dotted_allow = derive_dotted_resource_keys(root, resource_symbols, report) + + # --- Rule A: no stray dotted span-attribute keys ----------------------- + if l1_keys: + run_rule_a(keys_by_header, dotted_allow, report) + # --- Rule G: keys must be lower_snake_case ----------------------------- + if l1_keys: + run_rule_g(keys_by_header, report) + # --- Rule F (+ Rule H): scan telemetry call-sites ---------------------- + # Runs UNCONDITIONALLY: Rule F is a purely syntactic check (is this argument + # a literal?) and does not need the L1 key set, so a code path that uses + # SpanGuard::span/setAttribute directly without ever defining a *SpanNames.h + # is still caught. Rule H (warning) additionally flags constant references + # not defined in any *SpanNames.h. + header_symbols = spanname_symbol_names(headers) + run_rule_f(root, report, header_symbols) + + # --- Cross-layer rules B/C/D/E (each presence-gated) ------------------- + # L6 native-metric labels: span attributes are not the only valid dashboard + # labels — the MetricsRegistry emits OTel metrics whose label keys are an + # additional source of truth. Derive them dynamically (same principle as L1) + # so dashboards may reference them without tripping Rule D. + metric_labels = metric_label_names(root) + run_rule_b_collector(root, l1_keys, report) + run_rule_c_tempo(root, l1_keys, report) + run_rule_d_dashboards(root, l1_keys, metric_labels, report) + run_rule_e_runbook(root, l1_keys, report) + + report.render_and_exit() + + +def resource_create_block(text: str) -> str: + """Return the text inside the first `Resource::Create({ ... })` argument + list, brace-matched so nested `{key, value}` initializers are contained. + Empty string if the call is absent.""" + m = re.search(r"Resource::Create\(\s*\{", text) + if not m: + return "" + i = m.end() # one char past the opening `{` + depth, start = 1, i + while i < len(text) and depth > 0: + c = text[i] + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + i += 1 + return text[start : i - 1] + + +def derive_dotted_resource_keys( + root: Path, symbols: Dict[str, str], report: Report +) -> Set[str]: + """Legitimate dotted keys = ONLY the keys the code actually sets as RESOURCE + attributes, i.e. the entries inside Telemetry.cpp's `Resource::Create({...})` + call: the standard semconv keys (`service.*`) plus any `attr::` + constants passed there (resolved to their wire key via the global symbol + table, e.g. `attr::networkId` -> `xrpl.network.id`). + + A dotted key DECLARED in a `*SpanNames.h` header but NOT passed to + Resource::Create() is a span attribute wearing the resource form — a Rule-A + violation, never allowlisted. Deriving the allowlist from the actual + resource call (not from "any dotted key in the base header") is what lets + Rule A catch a stray dotted span attr such as `xrpl.ledger.hash`.""" + allow: Set[str] = set() + tele = root / "src" / "libxrpl" / "telemetry" / "Telemetry.cpp" + if not tele.is_file(): + report.skip("resource-derive", "Telemetry.cpp not present") + return allow + block = resource_create_block(read_source(tele)) + # semconv::::k -> the dotted OTel-standard key. The + # CamelKey already embeds the group, e.g. service::kServiceInstanceId + # -> service.instance.id. Split the CamelCase name into dotted lowercase + # segments; if it does not lead with the group, prepend the group. + for m in re.finditer(r"semconv::(\w+)::k(\w+)", block): + group, camel = m.group(1), m.group(2) + segments = camel_to_dotsegments(camel) + if segments and segments[0] == group: + allow.add(".".join(segments)) + else: + allow.add(group + "." + ".".join(segments)) + # attr:: constants set as resource attrs (e.g. networkId/networkType); + # resolve each to its wire key and allowlist only the dotted ones. + for m in re.finditer(r"attr::(\w+)", block): + val = symbols.get(m.group(1)) + if val is not None and "." in val: + allow.add(val) + report.ok(f"resource dotted-key allowlist derived: {sorted(allow)}") + return allow + + +def camel_to_dotsegments(s: str) -> List[str]: + """Split a CamelCase identifier into lowercase dot-segment parts, e.g. + `ServiceInstanceId` -> ['service', 'instance', 'id'].""" + return [w.lower() for w in re.findall(r"[A-Z][a-z0-9]*", s)] + + +def run_rule_a( + keys_by_header: Dict[Path, Set[str]], dotted_allow: Set[str], report: Report +) -> None: + """Any dotted attribute key that is not an allowed resource key is a + violation, reported against the header that declares it.""" + found = False + for h in sorted(keys_by_header): + for key in sorted(keys_by_header[h]): + if "." in key and key not in dotted_allow: + found = True + report.violation("A", h.name, key, "underscore form, not dotted") + if not found: + report.ok("A: no stray dotted span-attribute keys") + + +# A lower_snake_case identifier segment: starts lowercase, then lowercase / +# digits / underscores. No uppercase, no spaces, no camelCase. +SNAKE_SEGMENT = re.compile(r"^[a-z][a-z0-9_]*$") + + +def run_rule_g(keys_by_header: Dict[Path, Set[str]], report: Report) -> None: + """Every attribute key must be lower_snake_case. Bare/underscore keys must + match ^[a-z][a-z0-9_]*$; dotted resource keys must be lowercase + dot-separated segments (each segment lower_snake_case). Flags camelCase, + UPPERCASE, spaces, and other stray characters.""" + found = False + for h in sorted(keys_by_header): + for key in sorted(keys_by_header[h]): + segments = key.split(".") + if all(SNAKE_SEGMENT.match(seg) for seg in segments): + continue + found = True + report.violation("G", h.name, key, "must be lower_snake_case") + if not found: + report.ok("G: all attribute keys are lower_snake_case") + + +# Which argument positions of each call must be a constant (0-based). The +# attribute VALUE position is intentionally absent: values are runtime data +# (command names, hashes, counts), not naming-convention surface. +# setAttribute(key, value) -> check arg 0 (key); value (arg 1) exempt +# addEvent(name[, attrs]) -> check arg 0 (event name) +# span(category, prefix, name) -> check args 1,2 (prefix + span-name leaf) +# rootSpan(category, prefix, name)-> check args 1,2 (same signature as span) +# childSpan(name[, parentCtx]) -> check arg 0 (span-name leaf) +CONSTANT_ARG_POSITIONS: Dict[str, Set[int]] = { + "setAttribute": {0}, + "addEvent": {0}, + "span": {1, 2}, + "rootSpan": {1, 2}, # same signature as span(cat, prefix, name) + "childSpan": {0}, +} + + +def is_test_path(path: Path) -> bool: + """True if the path is test code. Tests legitimately pass arbitrary literal + keys/names to exercise the API mechanics, so Rule F does not apply to them. + Matches a `test`/`tests` directory anywhere in the path (e.g. src/test/, + src/tests/, .../detail/tests/).""" + return any(part in ("test", "tests") for part in path.parts) + + +# A constant reference passed at a call-site, e.g. `rpc_span::attr::command` +# or a bare `myKey`. We capture the leaf identifier (after the last `::`). +IDENTIFIER_ARG = re.compile(r"^[\s&*]*([A-Za-z_][\w:]*)\s*$") + + +def spanname_symbol_names(headers: List[Path]) -> Set[str]: + """Every `inline constexpr auto NAME = ...;` symbol defined across the + *SpanNames.h headers, by bare name. Used by Rule H to tell whether a + constant referenced at a call-site actually lives in a SpanNames header.""" + names: Set[str] = set() + for h in headers: + for m in CONST_DEF.finditer(strip_comments(read_source(h))): + names.add(m.group(1)) + return names + + +def run_rule_f(root: Path, report: Report, header_symbols: Set[str]) -> None: + """Walk every telemetry call-site (non-test, non-*SpanNames.h) and check the + constant-only argument positions of + setAttribute/addEvent/span/rootSpan/childSpan: + + Rule F (FAIL): a string literal in a key / span-name position. Attribute + VALUES are exempt (runtime data). + Rule H (WARN): a constant reference whose name is not defined in any + *SpanNames.h. The constant should live in the corresponding + *SpanNames.h (single source of truth); defining it in-place bypasses + the naming rules. Warn rather than fail — the argument may instead be a + legitimately dynamic local (e.g. a computed span-name leaf).""" + found_f = False + sources = [ + p + for base in ("src", "include") + for ext in ("*.h", "*.cpp") + for p in (root / base).rglob(ext) + if p.is_file() + ] + for path in sorted(sources): + if path.name.endswith("SpanNames.h") or is_test_path(path): + continue + text = read_source(path) + rel = path.relative_to(root) + for call, arglist, lineno in iter_calls(text): + positions = CONSTANT_ARG_POSITIONS.get(call, set()) + args = split_top_level_args(arglist) + for idx in positions: + if idx >= len(args): + continue + arg = args[idx] + lit = STRING_LITERAL.search(arg) + if lit: + found_f = True + report.violation( + "F", + f"{rel}:{lineno}", + f'{call} arg{idx} "{lit.group(1)}"', + "use a *SpanNames.h constant", + ) + continue + # Not a literal: Rule H warns when a NAMESPACE-QUALIFIED constant + # reference (e.g. `consensus::span::accept`) is not defined in + # any *SpanNames.h — i.e. the constant was defined in-place + # instead of in the proper header. We only consider qualified + # refs (containing `::`): a bare lowercase identifier is almost + # always a legitimately dynamic local (a computed span-name leaf + # or attribute value), not a misplaced constant, so warning on it + # would be noise. Standard-library types (std::...) are skipped. + ident = IDENTIFIER_ARG.match(arg) + if not (ident and header_symbols): + continue + ref = ident.group(1) + if "::" not in ref or ref.startswith("std::"): + continue + leaf = ref.split("::")[-1] + if leaf not in header_symbols: + report.warning( + "H", + f"{rel}:{lineno}", + f"{call} arg{idx} {ref}", + "not defined in any *SpanNames.h", + ) + if not found_f: + report.ok("F: no string-literal keys/names at telemetry call-sites") + + +def iter_calls(text: str): + """Yield (call_name, raw_arglist, lineno) for each setAttribute/addEvent/ + span/rootSpan/childSpan invocation, spanning multiple physical lines if + needed.""" + for m in CALLSITE.finditer(text): + name = m.group(1) + # Walk from the opening paren, balancing nesting to find the close. + # Parens inside a "string literal" are ignored so a value such as + # `setAttribute(k, ")")` does not close the call early. + i = m.end() # one char past the '(' + depth = 1 + in_str = False + escaped = False + while i < len(text) and depth > 0: + c = text[i] + if in_str: + if escaped: + escaped = False + elif c == "\\": + escaped = True + elif c == '"': + in_str = False + elif c == '"': + in_str = True + elif c == "(": + depth += 1 + elif c == ")": + depth -= 1 + i += 1 + arglist = text[m.end() : i - 1] + lineno = text.count("\n", 0, m.start()) + 1 + yield name, arglist, lineno + + +def run_rule_b_collector(root: Path, l1_keys: Set[str], report: Report) -> None: + path = root / "docker" / "telemetry" / "otel-collector-config.yaml" + if not path.is_file(): + report.skip("B", "collector config not present") + return + text = read_source(path) + if "spanmetrics" not in text: + report.skip("B", "no spanmetrics block in collector config") + return + dims = extract_spanmetrics_dimensions(text) + if not l1_keys: + report.skip("B", "no L1 key set to validate against") + return + miss = [d for d in dims if d not in l1_keys] + for d in miss: + report.violation("B", str(path.relative_to(root)), d, "must exist in L1") + if not miss: + report.ok(f"B: {len(dims)} collector dimension(s) all in L1") + + +def extract_spanmetrics_dimensions(text: str) -> List[str]: + dims: List[str] = [] + in_dims = False + for line in text.splitlines(): + if re.search(r"\bdimensions\s*:", line): + in_dims = True + continue + if in_dims: + m = re.search(r"-\s*name\s*:\s*([A-Za-z0-9_.]+)", line) + if m: + dims.append(m.group(1)) + elif line.strip() and not line.lstrip().startswith("-") and ":" in line: + in_dims = False + return dims + + +def run_rule_c_tempo(root: Path, l1_keys: Set[str], report: Report) -> None: + # The trace-search filter tags live in the Grafana Tempo DATASOURCE + # provisioning file (search.filters[].{tag,scope}); the Tempo server + # tempo.yaml has no such tags. Prefer the datasource file; fall back to the + # server file so the rule still does something if the layout changes. + candidates = [ + root / "docker/telemetry/grafana/provisioning/datasources/tempo.yaml", + root / "docker/telemetry/tempo.yaml", + ] + path = next((p for p in candidates if p.is_file()), None) + if path is None: + report.skip("C", "tempo datasource provisioning not present") + return + if not l1_keys: + report.skip("C", "no L1 key set to validate against") + return + # Pair each filter's `tag:` with its `scope:` (a few lines below it) and + # validate only span-scope tags — resource/intrinsic tags (service.*, name, + # status, duration) are not span attributes. Strip a TraceQL span. prefix. + lines = read_source(path).splitlines() + span_tags: List[str] = [] + for i, line in enumerate(lines): + m = re.search(r"^\s*tag:\s*(\S+)", line) + if not m: + continue + scope = next( + ( + sm.group(1) + for j in range(i, min(i + 4, len(lines))) + for sm in [re.search(r"scope:\s*(\S+)", lines[j])] + if sm + ), + "", + ) + if scope == "span": + span_tags.append(TRACEQL_SCOPE.sub("", m.group(1))) + if not span_tags: + report.skip("C", "no span-scope filter tags in tempo datasource") + return + miss = [t for t in span_tags if t not in l1_keys] + for t in sorted(set(miss)): + report.violation("C", str(path.relative_to(root)), t, "must exist in L1") + if not miss: + report.ok(f"C: {len(span_tags)} tempo span-filter tag(s) all in L1") + + +def metric_label_names(root: Path) -> Set[str]: + """L6: OTel native-metric label keys emitted by the telemetry code, e.g. + `counter->Add(1, {{"job_type", value}})` in MetricsRegistry.cpp. These are + a valid source of dashboard labels distinct from span attributes (L1).""" + labels: Set[str] = set() + for base in ("src", "include"): + for p in (root / base).rglob("*.cpp"): + if not p.is_file(): + continue + text = read_source(p) + if "MetricsRegistry" not in p.name and "metric" not in text.lower(): + continue + labels |= set(METRIC_LABEL.findall(text)) + return labels + + +# Identity labels stamped by EXTERNAL infrastructure the OTel pipeline in this +# repo does not own: the perf-iac harness attaches these to every metric it +# scrapes so dashboards can filter by which build/role produced a series. They +# have no L1 (*SpanNames.h), L2 (collector config), or L6 (MetricsRegistry.cpp) +# source to derive from, so — unlike every other dashboard label — they cannot +# be validated dynamically. This is a deliberate, narrow exception to the "no +# hardcoded allowlist" design principle, kept separate from the generic +# Prometheus/Grafana `builtins` set below so it stays visible and auditable. +# Add a label here ONLY if it is genuinely injected by infra outside this +# repo's OTel code (never as a workaround for a dashboard querying a label +# that nothing actually emits — that is a real Rule D violation). +EXTERNAL_INFRA_LABELS = { + "xrpl_work_item", # perf-iac: ticket/work-item id for the perf comparison run + "xrpl_branch", # perf-iac: git ref of the xrpld build under test + "xrpl_node_role", # perf-iac: validator/peer role in the perf cluster +} + + +def run_rule_d_dashboards( + root: Path, l1_keys: Set[str], metric_labels: Set[str], report: Report +) -> None: + dash_dir = root / "docker" / "telemetry" / "grafana" / "dashboards" + files = sorted(dash_dir.glob("*.json")) if dash_dir.is_dir() else [] + if not files: + report.skip("D", "no dashboard JSON present") + return + if not l1_keys: + report.skip("D", "no L1 key set to validate against") + return + builtins = { + "__name__", # Prometheus reserved label for the metric name itself + "le", + "exported_instance", + "span_name", + "status_code", + "service_name", + "service_version", + "service_instance_id", + "job", + "job_type", # standard Prometheus label for job-queue metrics + "instance", + } + # A dashboard label is valid if it is a span attribute (L1), a native-metric + # label (L6), a Prometheus/Grafana builtin, or an external-infra identity + # label (EXTERNAL_INFRA_LABELS). + valid = l1_keys | metric_labels | builtins | EXTERNAL_INFRA_LABELS + found = False + for f in files: + try: + text = read_source(f) + except OSError: + continue + # PromQL `sum by (a, b)` and `{label="..."}` references. + labels: Set[str] = set() + for m in re.finditer(r"by\s*\(([^)]*)\)", text): + labels |= {x.strip() for x in m.group(1).split(",") if x.strip()} + for m in re.finditer(r"\b([a-z_][a-z0-9_.]*)\s*[=!]~?\s*\"", text): + labels.add(m.group(1)) + for lbl in sorted(labels): + # Strip a TraceQL scope prefix (span./resource./...) — the bare + # attribute is what must resolve against L1. + bare = TRACEQL_SCOPE.sub("", lbl) + if bare in valid: + continue + found = True + report.violation( + "D", + str(f.relative_to(root)), + lbl, + "must exist in L1, a metric label, or be a builtin", + ) + if not found: + report.ok(f"D: dashboard PromQL labels all resolve ({len(files)} file(s))") + + +def run_rule_e_runbook(root: Path, l1_keys: Set[str], report: Report) -> None: + path = root / "docs" / "telemetry-runbook.md" + if not path.is_file(): + report.skip("E", "runbook not present") + return + if not l1_keys: + report.skip("E", "no L1 key set to validate against") + return + text = read_source(path) + found = False + # Only the dotted `xrpl..` attribute form is a violation. The + # `xrpl.`-with-trailing-dot anchor is the discriminator: it matches the old + # dotted attribute convention being migrated away from, while everything + # else legitimately dotted in the runbook does NOT match it — + # * span names (`consensus.round`, `tx.process`) no `xrpl.` prefix + # * filenames (`xrpld.cfg`, `RCLConsensus.cpp`) `xrpld.`/`.cpp`, not `xrpl.` + # * OTel-standard (`service.name`, `http.method`) no `xrpl.` prefix + # * metric labels (`xrpl_rpc_command`) underscore, no dot + # Legitimate dotted resource attrs (`xrpl.network.id`/`.type`) are in L1 and + # are skipped. A dotted `xrpl.` token absent from L1 is a genuine doc/code + # mismatch (e.g. `xrpl.tx.hash` where the code emits `tx_hash`). + # EXTERNAL_INFRA_LABELS (Rule D) holds the underscore/metric-label form of + # the perf-iac identity attrs; the resource-attribute layer stamps the same + # identities dotted (xrpl.work.item/.branch/.node.role -- see the alloy + # pipeline that owns them), so also skip a token whose dotted-to-underscore + # form is in that set. + external_infra_dotted = {lbl.replace("_", ".") for lbl in EXTERNAL_INFRA_LABELS} + for m in re.finditer(r"`(xrpl\.[a-z][a-z0-9_.]*)`", text): + token = m.group(1) + if token in l1_keys: # legitimate dotted resource attr (xrpl.network.*) + continue + if token in external_infra_dotted: # perf-iac resource-attribute layer + continue + found = True + report.violation( + "E", str(path.relative_to(root)), token, "underscore, not dotted" + ) + if not found: + report.ok("E: runbook attribute references consistent with L1") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/otel-naming/test_check_otel_naming.py b/.github/scripts/otel-naming/test_check_otel_naming.py new file mode 100644 index 00000000000..890648cefea --- /dev/null +++ b/.github/scripts/otel-naming/test_check_otel_naming.py @@ -0,0 +1,901 @@ +#!/usr/bin/env python3 + +"""Unit tests for check_otel_naming.py. + +Stdlib-only (unittest), matching the dependency-free policy of the check itself. +Run from anywhere: + + python .github/scripts/otel-naming/test_check_otel_naming.py + +Each rule is exercised in isolation against a synthetic tree / synthetic L1 key +set, covering positive (must flag), negative (must not flag), and boundary +cases. Rule E (runbook dotted-attribute detection) has the densest coverage +because its discriminator — the `xrpl..` prefix vs span names, +filenames, OTel-standard keys, and metric labels — is the subtlest. +""" + +import contextlib +import importlib.util +import io +import shutil +import tempfile +import unittest +from pathlib import Path + +# Load the check module by path (it is not an importable package). +_spec = importlib.util.spec_from_file_location( + "check_otel_naming", str(Path(__file__).with_name("check_otel_naming.py")) +) +chk = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(chk) + + +# A controlled L1 set used across tests: the two legitimate dotted resource +# attrs plus a handful of underscore span-attribute keys. +L1 = { + "xrpl.network.id", + "xrpl.network.type", + "tx_hash", + "peer_id", + "consensus_mode", + "command", + "rpc_status", + "ledger_seq", +} + + +def _run_rule_e(runbook_text: str): + """Run Rule E against a synthetic runbook; return the flagged tokens.""" + d = Path(tempfile.mkdtemp()) + try: + (d / "docs").mkdir() + (d / "docs" / "telemetry-runbook.md").write_text(runbook_text) + report = chk.Report() + chk.run_rule_e_runbook(d, set(L1), report) + return sorted(v[2] for v in report.violations) + finally: + shutil.rmtree(d) + + +class RuleERunbook(unittest.TestCase): + """Rule E: only dotted `xrpl..` attribute keys are flagged.""" + + # ----- positive: genuine dotted attribute-key violations ----- + def test_single_dotted_attr(self): + self.assertEqual(_run_rule_e("`xrpl.tx.hash`"), ["xrpl.tx.hash"]) + + def test_multiple_dotted_attrs(self): + self.assertEqual( + _run_rule_e("`xrpl.tx.hash` and `xrpl.consensus.mode`"), + ["xrpl.consensus.mode", "xrpl.tx.hash"], + ) + + def test_deep_dotted_three_segments(self): + self.assertEqual( + _run_rule_e("`xrpl.consensus.ledger.seq`"), ["xrpl.consensus.ledger.seq"] + ) + + def test_dotted_attr_with_underscore_field(self): + self.assertEqual( + _run_rule_e("`xrpl.consensus.round_id`"), ["xrpl.consensus.round_id"] + ) + + def test_repeated_token_reported_each_occurrence(self): + self.assertEqual( + _run_rule_e("`xrpl.tx.hash` ... `xrpl.tx.hash`"), + ["xrpl.tx.hash", "xrpl.tx.hash"], + ) + + def test_resource_attr_not_in_l1_is_flagged(self): + self.assertEqual( + _run_rule_e("`xrpl.network.unknown`"), ["xrpl.network.unknown"] + ) + + # ----- negative: legitimately-dotted tokens that must NOT be flagged ----- + def test_span_name_single(self): + self.assertEqual(_run_rule_e("`consensus.round`"), []) + + def test_span_name_multi_segment(self): + self.assertEqual( + _run_rule_e("`consensus.phase.open` `rpc.command.server_info`"), [] + ) + + def test_filename_cfg(self): + self.assertEqual(_run_rule_e("`xrpld.cfg`"), []) + + def test_filename_cpp(self): + self.assertEqual(_run_rule_e("`RCLConsensus.cpp`"), []) + + def test_otel_standard_service_name(self): + self.assertEqual(_run_rule_e("`service.name`"), []) + + def test_otel_standard_http_method(self): + self.assertEqual(_run_rule_e("`http.method`"), []) + + def test_metric_label_underscore(self): + self.assertEqual(_run_rule_e("`xrpl_rpc_command`"), []) + + def test_bare_underscore_attrs(self): + self.assertEqual(_run_rule_e("`tx_hash` `consensus_mode`"), []) + + def test_legit_dotted_resource_attrs_in_l1(self): + self.assertEqual(_run_rule_e("`xrpl.network.id` `xrpl.network.type`"), []) + + def test_external_infra_dotted_resource_attrs_not_flagged(self): + # perf-iac stamps these as dotted resource attrs (alloy pipeline); + # EXTERNAL_INFRA_LABELS (Rule D) holds their underscore metric-label + # form -- Rule E must also exempt the dotted resource-attr form. + self.assertEqual( + _run_rule_e("`xrpl.work.item` `xrpl.branch` `xrpl.node.role`"), [] + ) + + def test_prose_word(self): + self.assertEqual(_run_rule_e("the `command` attribute"), []) + + def test_plain_prose_no_backticks(self): + self.assertEqual(_run_rule_e("xrpl.tx.hash without backticks is prose"), []) + + # ----- boundary ----- + def test_empty_runbook(self): + self.assertEqual(_run_rule_e(""), []) + + def test_lookalike_prefix_xrpld(self): + # `xrpld.` is NOT `xrpl.` — must not match. + self.assertEqual(_run_rule_e("`xrpld.foo`"), []) + + def test_lookalike_prefix_underscore(self): + # `xrpl_rpc.command` starts with `xrpl_`, not `xrpl.`. + self.assertEqual(_run_rule_e("`xrpl_rpc.command`"), []) + + def test_uppercase_segment_not_matched(self): + # The pattern requires a lowercase char after `xrpl.`; uppercase keys are + # caught by Rule G at the L1 layer, not by the runbook text scan. + self.assertEqual(_run_rule_e("`xrpl.TX.hash`"), []) + + def test_token_touching_table_pipes(self): + self.assertEqual(_run_rule_e("| `xrpl.tx.hash` | desc |"), ["xrpl.tx.hash"]) + + def test_mixed_line_only_xrpl_dotted_flagged(self): + self.assertEqual( + _run_rule_e("`consensus.round` uses `xrpl.tx.hash` and `service.name`"), + ["xrpl.tx.hash"], + ) + + def test_skips_when_runbook_absent(self): + d = Path(tempfile.mkdtemp()) + try: + report = chk.Report() + chk.run_rule_e_runbook(d, set(L1), report) + self.assertEqual(report.violations, []) + self.assertTrue(any("SKIP: E" in s for s in report.skips)) + finally: + shutil.rmtree(d) + + def test_skips_when_l1_empty(self): + d = Path(tempfile.mkdtemp()) + try: + (d / "docs").mkdir() + (d / "docs" / "telemetry-runbook.md").write_text("`xrpl.tx.hash`") + report = chk.Report() + chk.run_rule_e_runbook(d, set(), report) + self.assertEqual(report.violations, []) + self.assertTrue(any("SKIP: E" in s for s in report.skips)) + finally: + shutil.rmtree(d) + + +class DslParser(unittest.TestCase): + """The makeStr/join/seg:: constexpr DSL resolver — the foundation of the + L1 key set. Covers flat, nested, cross-file, alias, and multi-line forms.""" + + def test_flat_join(self): + syms = chk.resolve_constants( + 'inline constexpr auto a = makeStr("xrpl");\n' + 'inline constexpr auto b = makeStr("network");\n' + "inline constexpr auto c = join(a, b);\n" + ) + self.assertEqual(syms["c"], "xrpl.network") + + def test_nested_join_three_segments(self): + syms = chk.resolve_constants( + 'inline constexpr auto xrpl = makeStr("xrpl");\n' + 'inline constexpr auto network = makeStr("network");\n' + "inline constexpr auto networkId = " + 'join(join(xrpl, network), makeStr("id"));\n' + ) + self.assertEqual(syms["networkId"], "xrpl.network.id") + + def test_qualified_seg_reference(self): + # `seg::rpc` resolves by its bare leaf `rpc`. + syms = chk.resolve_constants('inline constexpr auto rpc = makeStr("rpc");\n') + syms2 = chk.resolve_constants( + 'inline constexpr auto command = join(seg::rpc, makeStr("command"));\n', + syms, + ) + self.assertEqual(syms2["command"], "rpc.command") + + def test_alias_reference(self): + syms = chk.resolve_constants('inline constexpr auto rpc = makeStr("rpc");\n') + chk.resolve_constants("inline constexpr auto alias = seg::rpc;\n", syms) + self.assertEqual(syms["alias"], "rpc") + + def test_unresolvable_expr_omitted(self): + syms = chk.resolve_constants("inline constexpr auto x = join(unknown, y);\n") + self.assertNotIn("x", syms) + + def test_split_top_level_args_respects_nesting(self): + self.assertEqual( + chk.split_top_level_args("join(seg::a, b), c"), + ["join(seg::a, b)", " c"], + ) + + def test_split_top_level_args_ignores_comma_in_string(self): + self.assertEqual( + chk.split_top_level_args('key, ","'), + ["key", ' ","'], + ) + + def test_strip_comments_removes_line_and_block(self): + self.assertEqual( + chk.strip_comments("a // line\nb /* blk */ c").split(), + ["a", "b", "c"], + ) + + +def _write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + + +def _header(ns_attr_body: str, prefix_seg: str = "") -> str: + """A minimal *SpanNames.h body: optional seg defs + a namespace attr block.""" + return ( + "#pragma once\n" + + prefix_seg + + "namespace xrpl::telemetry::demo::span {\n" + + "namespace attr {\n" + + ns_attr_body + + "} // namespace attr\n" + + "}\n" + ) + + +class AttrKeyExtraction(unittest.TestCase): + """attr_keys_from_header: comment-stripping + multi-line + using re-export.""" + + def _l1(self, header_text): + d = Path(tempfile.mkdtemp()) + try: + h = d / "src" / "DemoSpanNames.h" + _write(h, header_text) + syms = chk.build_global_symbols([h]) + return chk.attr_keys_from_header(h, syms) + finally: + shutil.rmtree(d) + + def test_single_line_makestr(self): + keys = self._l1(_header('inline constexpr auto k = makeStr("tx_hash");\n')) + self.assertIn("tx_hash", keys) + + def test_multiline_constexpr_captured(self): + keys = self._l1( + _header("inline constexpr auto k =\n" ' makeStr("round_time_ms");\n') + ) + self.assertIn("round_time_ms", keys) + + def test_commented_makestr_not_leaked(self): + keys = self._l1( + _header( + 'inline constexpr auto k = makeStr("good");\n' + '// inline constexpr auto bad = makeStr("old.dotted");\n' + ) + ) + self.assertIn("good", keys) + self.assertNotIn("old.dotted", keys) + + def test_block_commented_makestr_not_leaked(self): + keys = self._l1( + _header( + 'inline constexpr auto k = makeStr("good");\n' + '/* makeStr("blockbad") */\n' + ) + ) + self.assertNotIn("blockbad", keys) + + +class CamelToDotSegments(unittest.TestCase): + """semconv CamelCase -> dotted OTel-standard key derivation.""" + + def test_service_instance_id(self): + self.assertEqual( + chk.camel_to_dotsegments("ServiceInstanceId"), + ["service", "instance", "id"], + ) + + def test_service_name(self): + self.assertEqual(chk.camel_to_dotsegments("ServiceName"), ["service", "name"]) + + def test_derive_keys_from_telemetry_cpp(self): + d = Path(tempfile.mkdtemp()) + try: + tele = d / "src" / "libxrpl" / "telemetry" / "Telemetry.cpp" + _write( + tele, + "resource::Resource::Create({\n" + " {semconv::service::kServiceName, x},\n" + " {semconv::service::kServiceInstanceId, y},\n" + "});\n", + ) + report = chk.Report() + allow = chk.derive_dotted_resource_keys(d, {}, report) + self.assertIn("service.name", allow) + self.assertIn("service.instance.id", allow) + finally: + shutil.rmtree(d) + + +class SymbolCollision(unittest.TestCase): + """attr_keys_from_header must resolve a constant against ITS OWN header, so + two headers defining a same-named constant each report their real wire key. + Regression for the flat-symbol-table collision that let a later header + clobber an earlier one and erased a dotted key from L1 (a Rule-A blind + spot).""" + + def _build(self, files): + d = Path(tempfile.mkdtemp()) + paths = {} + for rel, text in files.items(): + p = d / rel + _write(p, text) + paths[rel] = p + return d, paths + + def test_same_named_const_not_clobbered_across_headers(self): + base = ( + "#pragma once\n" + "namespace xrpl::telemetry {\n" + 'namespace seg { inline constexpr auto xrpl = makeStr("xrpl");\n' + 'inline constexpr auto ledger = makeStr("ledger"); }\n' + "namespace attr {\n" + "inline constexpr auto ledgerHash = " + 'join(join(seg::xrpl, seg::ledger), makeStr("hash"));\n' + "}\n}\n" + ) + cons = ( + "#pragma once\n" + "namespace xrpl::telemetry::consensus::span {\n" + "namespace attr { inline constexpr auto ledgerHash = " + 'makeStr("ledger_hash"); }\n}\n' + ) + d, paths = self._build( + { + "include/xrpl/telemetry/SpanNames.h": base, + "src/xrpld/consensus/ConsensusSpanNames.h": cons, + } + ) + try: + headers = chk.find_spanname_headers(d) + syms = chk.build_global_symbols(headers) + by_name = {p.name: chk.attr_keys_from_header(p, syms) for p in headers} + # The base header keeps its dotted key; consensus keeps the bare one. + self.assertIn("xrpl.ledger.hash", by_name["SpanNames.h"]) + self.assertEqual(by_name["ConsensusSpanNames.h"], {"ledger_hash"}) + finally: + shutil.rmtree(d) + + def test_using_reexport_still_resolves_globally(self): + # A `using`-re-export imports a constant defined elsewhere; it must + # resolve against the global table, not the local header. + base = ( + "#pragma once\n" + "namespace xrpl::telemetry {\n" + "namespace attr { inline constexpr auto txHash = " + 'makeStr("tx_hash"); }\n}\n' + ) + dom = ( + "#pragma once\n" + "namespace xrpl::telemetry::tx::span {\n" + "namespace attr { using ::xrpl::telemetry::attr::txHash; }\n}\n" + ) + d, paths = self._build( + { + "include/xrpl/telemetry/SpanNames.h": base, + "src/xrpld/app/misc/TxSpanNames.h": dom, + } + ) + try: + headers = chk.find_spanname_headers(d) + syms = chk.build_global_symbols(headers) + keys = chk.attr_keys_from_header( + paths["src/xrpld/app/misc/TxSpanNames.h"], syms + ) + self.assertEqual(keys, {"tx_hash"}) + finally: + shutil.rmtree(d) + + +class ResourceAllowlistScope(unittest.TestCase): + """derive_dotted_resource_keys must allowlist ONLY the dotted keys actually + passed to Resource::Create() — not every dotted key in the base header. A + dotted attr declared in a header but not set as a resource attr is a Rule-A + violation.""" + + def _derive(self, tele_text, span_text): + d = Path(tempfile.mkdtemp()) + try: + _write(d / "src" / "libxrpl" / "telemetry" / "Telemetry.cpp", tele_text) + _write(d / "include" / "xrpl" / "telemetry" / "SpanNames.h", span_text) + headers = chk.find_spanname_headers(d) + syms = chk.build_global_symbols(headers) + allow = chk.derive_dotted_resource_keys(d, syms, chk.Report()) + return allow, syms, headers, d + except Exception: + shutil.rmtree(d) + raise + + def test_dotted_span_attr_not_allowlisted_and_flagged(self): + span = ( + "#pragma once\n" + "namespace xrpl::telemetry {\n" + 'namespace seg { inline constexpr auto xrpl = makeStr("xrpl");\n' + 'inline constexpr auto ledger = makeStr("ledger");\n' + 'inline constexpr auto network = makeStr("network"); }\n' + "namespace attr {\n" + "inline constexpr auto networkId = " + 'join(join(seg::xrpl, seg::network), makeStr("id"));\n' + "inline constexpr auto ledgerHash = " + 'join(join(seg::xrpl, seg::ledger), makeStr("hash"));\n' + "}\n}\n" + ) + tele = ( + "auto r = resource::Resource::Create({\n" + " {semconv::service::kServiceName, x},\n" + " {std::string(attr::networkId), n},\n" + "});\n" + ) + allow, syms, headers, d = self._derive(tele, span) + try: + # networkId IS a resource attr; ledgerHash is NOT, despite living in + # the base header. + self.assertIn("xrpl.network.id", allow) + self.assertNotIn("xrpl.ledger.hash", allow) + kbh = {h: chk.attr_keys_from_header(h, syms) for h in headers} + report = chk.Report() + chk.run_rule_a(kbh, allow, report) + self.assertEqual([v[2] for v in report.violations], ["xrpl.ledger.hash"]) + finally: + shutil.rmtree(d) + + def test_resource_block_brace_matched(self): + # A nested {key,value} initializer must not truncate the block scan. + tele = ( + "auto r = resource::Resource::Create({\n" + " {semconv::service::kServiceName, x},\n" + " {std::string(attr::networkType), t},\n" + "});\n" + ) + span = ( + "#pragma once\n" + "namespace xrpl::telemetry {\n" + 'namespace seg { inline constexpr auto xrpl = makeStr("xrpl");\n' + 'inline constexpr auto network = makeStr("network"); }\n' + "namespace attr { inline constexpr auto networkType = " + 'join(join(seg::xrpl, seg::network), makeStr("type")); }\n}\n' + ) + allow, _syms, _headers, d = self._derive(tele, span) + try: + self.assertIn("xrpl.network.type", allow) + self.assertIn("service.name", allow) + finally: + shutil.rmtree(d) + + +def _run_rule_a(keys_by_header, allow): + report = chk.Report() + chk.run_rule_a(keys_by_header, allow, report) + return sorted(v[2] for v in report.violations) + + +class RuleADotted(unittest.TestCase): + def test_dotted_attr_not_in_allow_flagged(self): + kbh = {Path("src/RpcSpanNames.h"): {"xrpl.tx.hash", "command"}} + self.assertEqual(_run_rule_a(kbh, {"xrpl.network.id"}), ["xrpl.tx.hash"]) + + def test_resource_attr_in_allow_passes(self): + kbh = {Path("src/SpanNames.h"): {"xrpl.network.id"}} + self.assertEqual(_run_rule_a(kbh, {"xrpl.network.id"}), []) + + def test_bare_key_never_flagged(self): + kbh = {Path("src/TxSpanNames.h"): {"tx_hash", "command"}} + self.assertEqual(_run_rule_a(kbh, set()), []) + + +def _run_rule_g(keys_by_header): + report = chk.Report() + chk.run_rule_g(keys_by_header, report) + return sorted(v[2] for v in report.violations) + + +class RuleGSnakeCase(unittest.TestCase): + def test_camelcase_flagged(self): + self.assertEqual(_run_rule_g({Path("h"): {"txHash"}}), ["txHash"]) + + def test_uppercase_flagged(self): + self.assertEqual(_run_rule_g({Path("h"): {"TX_HASH"}}), ["TX_HASH"]) + + def test_space_flagged(self): + self.assertEqual(_run_rule_g({Path("h"): {"bad key"}}), ["bad key"]) + + def test_snake_case_passes(self): + self.assertEqual(_run_rule_g({Path("h"): {"tx_hash", "rpc_status"}}), []) + + def test_dotted_resource_segments_pass(self): + self.assertEqual(_run_rule_g({Path("h"): {"xrpl.network.id"}}), []) + + def test_dotted_with_bad_segment_flagged(self): + self.assertEqual( + _run_rule_g({Path("h"): {"xrpl.Network.id"}}), ["xrpl.Network.id"] + ) + + +class RuleFAndH(unittest.TestCase): + """run_rule_f: literal keys/span-names flagged; values & tests exempt. + Rule H: qualified constant not in any header warns (non-fatal).""" + + def _run(self, rel_path, source, header_symbols=frozenset()): + d = Path(tempfile.mkdtemp()) + try: + _write(d / rel_path, source) + report = chk.Report() + chk.run_rule_f(d, report, set(header_symbols)) + return ( + sorted(v[2] for v in report.violations), + sorted(w[2] for w in report.warnings), + ) + finally: + shutil.rmtree(d) + + def test_literal_key_flagged(self): + v, _ = self._run("src/Foo.cpp", 'g.setAttribute("lit_key", v);\n') + self.assertEqual(v, ['setAttribute arg0 "lit_key"']) + + def test_literal_value_exempt(self): + v, _ = self._run("src/Foo.cpp", 'g.setAttribute(attr::command, "submit");\n') + self.assertEqual(v, []) + + def test_span_name_args_flagged(self): + v, _ = self._run("src/Foo.cpp", 'SpanGuard::span(cat, "rpc", "command");\n') + self.assertEqual(v, ['span arg1 "rpc"', 'span arg2 "command"']) + + def test_rootspan_literal_flagged_by_rule_f(self): + # rootSpan(cat, prefix, name) shares span()'s signature, so a string + # literal in the prefix/name position must FAIL rule F exactly as it + # does for span() — otherwise a call switched to rootSpan silently + # escapes span-name validation. + v, _ = self._run( + "src/Foo.cpp", + 'SpanGuard::rootSpan(cat, "peer", "validation.receive");\n', + ) + self.assertEqual( + v, ['rootSpan arg1 "peer"', 'rootSpan arg2 "validation.receive"'] + ) + + def test_rootspan_constant_args_accepted(self): + # Constant references in the prefix/name position are accepted (no + # rule F), mirroring span()'s constant-arg handling. + v, _ = self._run( + "src/Foo.cpp", + "SpanGuard::rootSpan(TraceCategory::Peer, seg::peer, " + "peer_span::op::validationReceive);\n", + ) + self.assertEqual(v, []) + + def test_test_path_exempt(self): + v, _ = self._run("src/test/Foo.cpp", 'g.setAttribute("lit_key", v);\n') + self.assertEqual(v, []) + + def test_spannames_header_exempt(self): + v, _ = self._run("src/DemoSpanNames.h", 'g.setAttribute("lit_key", v);\n') + self.assertEqual(v, []) + + def test_bare_span_call_not_matched(self): + # No SpanGuard/./-> receiver -> not a telemetry call-site. + v, _ = self._run("src/Foo.cpp", 'auto s = span("not", "telemetry");\n') + self.assertEqual(v, []) + + def test_multiline_call_reports_first_line(self): + v, _ = self._run("src/Foo.cpp", 'g.setAttribute(\n "k",\n v);\n') + self.assertEqual(v, ['setAttribute arg0 "k"']) + + def test_paren_in_string_value_does_not_break_parsing(self): + # The ")" inside the value must not end the call early; key still seen. + v, _ = self._run("src/Foo.cpp", 'g.setAttribute("k", ")");\n') + self.assertEqual(v, ['setAttribute arg0 "k"']) + + def test_rule_h_qualified_constant_warns(self): + v, w = self._run( + "src/Foo.cpp", + "g.setAttribute(consensus::span::accept, v);\n", + header_symbols={"command"}, + ) + self.assertEqual(v, []) + self.assertEqual(w, ["setAttribute arg0 consensus::span::accept"]) + + def test_rule_h_known_constant_no_warning(self): + _, w = self._run( + "src/Foo.cpp", + "g.setAttribute(rpc_span::attr::command, v);\n", + header_symbols={"command"}, + ) + self.assertEqual(w, []) + + def test_rule_h_bare_local_no_warning(self): + _, w = self._run( + "src/Foo.cpp", "g.setAttribute(myLeaf, v);\n", header_symbols={"command"} + ) + self.assertEqual(w, []) + + +class RuleBCollector(unittest.TestCase): + def _run(self, yaml_text, l1): + d = Path(tempfile.mkdtemp()) + try: + _write(d / "docker" / "telemetry" / "otel-collector-config.yaml", yaml_text) + report = chk.Report() + chk.run_rule_b_collector(d, set(l1), report) + return sorted(v[2] for v in report.violations), report.skips + finally: + shutil.rmtree(d) + + def test_dimension_not_in_l1_flagged(self): + y = "spanmetrics:\n dimensions:\n - name: bogus_dim\n - name: command\n" + v, _ = self._run(y, {"command"}) + self.assertEqual(v, ["bogus_dim"]) + + def test_all_dimensions_in_l1_pass(self): + y = "spanmetrics:\n dimensions:\n - name: command\n - name: rpc_status\n" + v, _ = self._run(y, {"command", "rpc_status"}) + self.assertEqual(v, []) + + def test_skip_when_no_spanmetrics_block(self): + v, skips = self._run("receivers:\n otlp:\n", {"command"}) + self.assertEqual(v, []) + self.assertTrue(any("SKIP: B" in s for s in skips)) + + +class RuleCTempo(unittest.TestCase): + """Rule C reads the Grafana Tempo DATASOURCE file's search.filters and + validates only span-scope tags against L1.""" + + DS = "docker/telemetry/grafana/provisioning/datasources/tempo.yaml" + + def _run(self, yaml_text, l1): + d = Path(tempfile.mkdtemp()) + try: + _write(d / self.DS, yaml_text) + report = chk.Report() + chk.run_rule_c_tempo(d, set(l1), report) + return sorted(v[2] for v in report.violations), report.skips + finally: + shutil.rmtree(d) + + def _filter(self, fid, tag, scope): + return ( + f" - id: {fid}\n" + f" tag: {tag}\n" + f' operator: "="\n' + f" scope: {scope}\n" + f" type: static\n" + ) + + def test_span_tag_not_in_l1_flagged(self): + y = "search:\n filters:\n" + self._filter("f1", "bogus_tag", "span") + v, _ = self._run(y, {"command"}) + self.assertEqual(v, ["bogus_tag"]) + + def test_span_tags_in_l1_pass(self): + y = ( + "search:\n filters:\n" + + self._filter("f1", "command", "span") + + self._filter("f2", "tx_hash", "span") + ) + v, _ = self._run(y, {"command", "tx_hash"}) + self.assertEqual(v, []) + + def test_resource_and_intrinsic_tags_ignored(self): + # service.* (resource) and name/status/duration (intrinsic) are not + # span attributes — they must not be validated against L1. + y = ( + "search:\n filters:\n" + + self._filter("f1", "service.instance.id", "resource") + + self._filter("f2", "name", "intrinsic") + + self._filter("f3", "duration", "intrinsic") + ) + v, skips = self._run(y, {"command"}) + self.assertEqual(v, []) + self.assertTrue(any("SKIP: C" in s for s in skips)) + + def test_skip_when_datasource_absent(self): + d = Path(tempfile.mkdtemp()) + try: + report = chk.Report() + chk.run_rule_c_tempo(d, {"command"}, report) + self.assertEqual(report.violations, []) + self.assertTrue(any("SKIP: C" in s for s in report.skips)) + finally: + shutil.rmtree(d) + + +class RuleDDashboards(unittest.TestCase): + def _run(self, json_text, l1, metric_labels=frozenset()): + d = Path(tempfile.mkdtemp()) + try: + _write( + d / "docker" / "telemetry" / "grafana" / "dashboards" / "x.json", + json_text, + ) + report = chk.Report() + chk.run_rule_d_dashboards(d, set(l1), set(metric_labels), report) + return sorted(v[2] for v in report.violations) + finally: + shutil.rmtree(d) + + def test_unknown_promql_label_flagged(self): + self.assertEqual( + self._run('"expr": "sum by (bogus_label) (x)"', {"command"}), + ["bogus_label"], + ) + + def test_builtin_labels_not_flagged(self): + self.assertEqual( + self._run('"expr": "sum by (le, span_name, exported_instance) (x)"', set()), + [], + ) + + def test_external_infra_labels_not_flagged(self): + # EXTERNAL_INFRA_LABELS (perf-iac identity labels with no in-tree + # source) must be recognized as valid, distinct from `builtins`. + expr = "sum by (" + ", ".join(sorted(chk.EXTERNAL_INFRA_LABELS)) + ") (x)" + self.assertEqual(self._run(f'"expr": "{expr}"', set()), []) + + def test_prometheus_name_label_not_flagged(self): + # `__name__` is the Prometheus reserved metric-name label; the renamed + # system-*.json dashboards use `sum by (le, __name__)`. + self.assertEqual( + self._run('"expr": "sum by (le, __name__) (rate(x[5m]))"', set()), + [], + ) + + def test_l1_label_passes(self): + self.assertEqual(self._run('"q": "{command=\\"x\\"}"', {"command"}), []) + + def test_traceql_span_prefix_stripped(self): + # `span.establish_count` must validate against the bare L1 key. + self.assertEqual( + self._run( + '"expr": "count_over_time(x) by (span.establish_count)"', + {"establish_count"}, + ), + [], + ) + + def test_traceql_resource_prefix_stripped(self): + self.assertEqual(self._run('"q": "{resource.service_name=\\"x\\"}"', set()), []) + + def test_native_metric_label_passes(self): + # `job_type` / `reason` are emitted by MetricsRegistry, not span attrs. + self.assertEqual( + self._run( + '"expr": "sum by (job_type, reason) (x)"', + {"command"}, + metric_labels={"job_type", "reason"}, + ), + [], + ) + + def test_unknown_label_still_flagged_with_metric_labels(self): + # A label that is neither L1, metric label, nor builtin still fails. + self.assertEqual( + self._run( + '"expr": "sum by (bogus) (x)"', + {"command"}, + metric_labels={"job_type"}, + ), + ["bogus"], + ) + + def test_span_prefixed_unknown_still_flagged(self): + # `span.not_a_key` whose bare form is unknown is still a violation. + self.assertEqual( + self._run('"expr": "x by (span.not_a_key)"', {"command"}), + ["span.not_a_key"], + ) + + +class MetricLabelExtraction(unittest.TestCase): + """L6: native-metric label keys parsed from C++ instrument calls.""" + + def test_extracts_add_label(self): + d = Path(tempfile.mkdtemp()) + try: + _write( + d / "src" / "xrpld" / "telemetry" / "MetricsRegistry.cpp", + 'counter->Add(1, {{"job_type", std::string(jobType)}});\n' + 'c2->Add(1, {{"reason", std::string(r)}});\n', + ) + self.assertEqual(chk.metric_label_names(d), {"job_type", "reason"}) + finally: + shutil.rmtree(d) + + def test_no_metrics_file_empty(self): + d = Path(tempfile.mkdtemp()) + try: + (d / "src").mkdir() + self.assertEqual(chk.metric_label_names(d), set()) + finally: + shutil.rmtree(d) + + +class ReportExitContract(unittest.TestCase): + @staticmethod + def _exit_code(report): + """Call render_and_exit (which prints + raises SystemExit), swallowing + its stdout, and return the exit code.""" + with contextlib.redirect_stdout(io.StringIO()): + try: + report.render_and_exit() + except SystemExit as e: + return e.code + return None # pragma: no cover - render_and_exit always exits + + def test_violation_exits_nonzero(self): + r = chk.Report() + r.violation("A", "f", "tok", "exp") + self.assertEqual(self._exit_code(r), 1) + + def test_clean_exits_zero(self): + r = chk.Report() + r.ok("all good") + self.assertEqual(self._exit_code(r), 0) + + def test_warning_only_exits_zero(self): + r = chk.Report() + r.warning("H", "f", "tok", "note") + self.assertEqual(self._exit_code(r), 0) + + +class RuleEReportTuple(unittest.TestCase): + """Assert Rule E records the full (rule, expected) tuple, not just token.""" + + def test_violation_tuple_fields(self): + d = Path(tempfile.mkdtemp()) + try: + (d / "docs").mkdir() + (d / "docs" / "telemetry-runbook.md").write_text("`xrpl.tx.hash`") + report = chk.Report() + chk.run_rule_e_runbook(d, {"xrpl.network.id"}, report) + self.assertEqual(len(report.violations), 1) + rule, _loc, token, expected = report.violations[0] + self.assertEqual(rule, "E") + self.assertEqual(token, "xrpl.tx.hash") + self.assertEqual(expected, "underscore, not dotted") + finally: + shutil.rmtree(d) + + def test_clean_runbook_records_ok(self): + d = Path(tempfile.mkdtemp()) + try: + (d / "docs").mkdir() + (d / "docs" / "telemetry-runbook.md").write_text( + "`tx_hash` `consensus.round`" + ) + report = chk.Report() + chk.run_rule_e_runbook(d, {"tx_hash"}, report) + self.assertEqual(report.violations, []) + self.assertTrue(any("E:" in c for c in report.checked)) + finally: + shutil.rmtree(d) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index 442a202a444..28718bfa91d 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -70,8 +70,10 @@ jobs: files: | # These paths are unique to `on-pr.yml`. .github/scripts/levelization/** + .github/scripts/otel-naming/** .github/scripts/rename/** .github/workflows/reusable-check-levelization.yml + .github/workflows/reusable-check-otel-naming.yml .github/workflows/reusable-check-rename.yml .github/workflows/on-pr.yml @@ -130,6 +132,11 @@ jobs: if: ${{ needs.should-run.outputs.go == 'true' }} uses: ./.github/workflows/reusable-check-levelization.yml + check-otel-naming: + needs: should-run + if: ${{ needs.should-run.outputs.go == 'true' }} + uses: ./.github/workflows/reusable-check-otel-naming.yml + check-rename: needs: should-run if: ${{ needs.should-run.outputs.go == 'true' }} @@ -200,6 +207,7 @@ jobs: if: failure() || cancelled() needs: - check-levelization + - check-otel-naming - check-rename - clang-tidy - build-test diff --git a/.github/workflows/reusable-check-otel-naming.yml b/.github/workflows/reusable-check-otel-naming.yml new file mode 100644 index 00000000000..a7af2da8cd6 --- /dev/null +++ b/.github/workflows/reusable-check-otel-naming.yml @@ -0,0 +1,28 @@ +# This workflow checks that OpenTelemetry span-attribute names stay consistent +# across the code (*SpanNames.h), collector, Tempo, dashboards, and docs. +# See .github/scripts/otel-naming/check_otel_naming.py and the +# "Telemetry span attribute naming" section in CONTRIBUTING.md. +name: Check OTel naming + +# This workflow can only be triggered by other workflows. +on: workflow_call + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-otel-naming + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + otel-naming: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - name: Check OTel naming + # The script is stdlib-only and reads only files already in the tree; + # it enforces each rule only when the layer it needs is present, so it + # works whether telemetry changes land in one PR or several. + run: python .github/scripts/otel-naming/check_otel_naming.py diff --git a/.github/workflows/telemetry-validation.yml b/.github/workflows/telemetry-validation.yml new file mode 100644 index 00000000000..e65505bf881 --- /dev/null +++ b/.github/workflows/telemetry-validation.yml @@ -0,0 +1,310 @@ +# Telemetry Validation CI Workflow +# +# Builds rippled with telemetry enabled, runs the multi-node workload +# harness, validates all telemetry data, and runs performance benchmarks. +# +# This is a separate workflow from the main CI. It runs: +# - On manual dispatch (workflow_dispatch) +# - On pushes to telemetry-related branches +# +# The workflow is intentionally heavyweight (builds rippled, starts Docker +# services, runs a multi-node cluster) — it validates the full telemetry +# stack end-to-end rather than individual unit tests. +# +# Architecture: two jobs to leverage cached dependencies: +# 1. build-xrpld — runs on a self-hosted runner inside the same container +# image the main CI uses (debian-bookworm-gcc-13). This ensures Conan +# packages are fetched from the XRPLF remote instead of built from +# source, and ccache hits the remote cache. +# 2. validate-telemetry — runs on ubuntu-latest (which has Docker) to +# launch the telemetry stack (OTel collector, Prometheus, Tempo, etc.) +# and validate the full pipeline end-to-end. + +name: Telemetry Validation + +on: + workflow_dispatch: + inputs: + rpc_rate: + description: "RPC load rate (requests per second)" + required: false + default: "50" + rpc_duration: + description: "RPC load duration (seconds)" + required: false + default: "120" + tx_tps: + description: "Transaction submit rate (TPS)" + required: false + default: "5" + tx_duration: + description: "Transaction submit duration (seconds)" + required: false + default: "120" + run_benchmark: + description: "Run performance benchmarks" + required: false + type: boolean + default: false + + push: + branches: + - "pratik/otel-phase*" + - "feature/otel-*" + - "feature/telemetry-*" + paths: + - ".github/workflows/telemetry-validation.yml" + - "docker/telemetry/**" + - "include/xrpl/basics/Telemetry*.h" + - "src/xrpld/app/misc/Telemetry*" + +concurrency: + group: telemetry-validation-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + BUILD_DIR: build + +jobs: + # ── Job 1: Build xrpld in the same container the main CI uses ────── + # This ensures Conan binary packages are fetched from the XRPLF remote + # (matching package IDs) and ccache hits the remote compilation cache. + build-xrpld: + name: Build xrpld + runs-on: [self-hosted, Linux, X64, heavy] + container: ghcr.io/xrplf/ci/debian-bookworm:gcc-13-sha-ab4d1f0 + timeout-minutes: 60 + env: + CCACHE_NAMESPACE: telemetry-validation + CCACHE_REMOTE_ONLY: true + CCACHE_REMOTE_STORAGE: http://cache.dev.ripplex.io:8080|layout=bazel + CCACHE_SLOPPINESS: include_file_ctime,include_file_mtime + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Prepare runner + uses: XRPLF/actions/prepare-runner@90f11ee655d1687824fb8793db770477d52afbab + with: + enable_ccache: ${{ github.repository_owner == 'XRPLF' }} + + - name: Print build environment + uses: XRPLF/actions/print-build-env@59dec886e4afb05a1724443af08baccbc045b574 + + - name: Get number of processors + uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf + id: nproc + with: + subtract: 2 + + - name: Setup Conan + uses: ./.github/actions/setup-conan + + - name: Build dependencies + uses: ./.github/actions/build-deps + with: + build_nproc: ${{ steps.nproc.outputs.nproc }} + build_type: Release + log_verbosity: verbose + + - name: Configure CMake + working-directory: ${{ env.BUILD_DIR }} + run: | + cmake \ + -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ + -DCMAKE_BUILD_TYPE=Release \ + .. + + - name: Build xrpld + working-directory: ${{ env.BUILD_DIR }} + env: + BUILD_NPROC: ${{ steps.nproc.outputs.nproc }} + run: | + cmake \ + --build . \ + --config Release \ + --parallel "${BUILD_NPROC}" \ + --target xrpld + + - name: Show ccache statistics + if: ${{ github.repository_owner == 'XRPLF' }} + run: ccache --show-stats -vv + + - name: Upload xrpld binary + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: xrpld-telemetry + path: ${{ env.BUILD_DIR }}/xrpld + retention-days: 1 + if-no-files-found: error + + # ── Job 2: Run telemetry validation on ubuntu-latest (has Docker) ── + validate-telemetry: + name: Telemetry Stack Validation + needs: build-xrpld + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Python dependencies + run: pip3 install -r docker/telemetry/workload/requirements.txt + + - name: Download xrpld binary + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + with: + name: xrpld-telemetry + path: ${{ env.BUILD_DIR }} + + - name: Make binaries and scripts executable + run: | + chmod +x ${{ env.BUILD_DIR }}/xrpld + chmod +x docker/telemetry/workload/*.sh + + - name: Run full telemetry validation + id: validation + env: + RPC_RATE: ${{ github.event.inputs.rpc_rate || '50' }} + RPC_DURATION: ${{ github.event.inputs.rpc_duration || '120' }} + TX_TPS: ${{ github.event.inputs.tx_tps || '5' }} + TX_DURATION: ${{ github.event.inputs.tx_duration || '120' }} + RUN_BENCHMARK: ${{ github.event.inputs.run_benchmark }} + run: | + ARGS="--xrpld ${{ env.BUILD_DIR }}/xrpld --skip-loki" + ARGS="$ARGS --rpc-rate $RPC_RATE" + ARGS="$ARGS --rpc-duration $RPC_DURATION" + ARGS="$ARGS --tx-tps $TX_TPS" + ARGS="$ARGS --tx-duration $TX_DURATION" + if [ "$RUN_BENCHMARK" = "true" ]; then + ARGS="$ARGS --with-benchmark" + fi + docker/telemetry/workload/run-full-validation.sh $ARGS + # continue-on-error allows subsequent steps (artifact upload, + # summary printing) to run even if validation fails. The final + # "Check validation result" step re-checks steps.validation.outcome + # (the pre-continue-on-error result) and fails the job properly. + continue-on-error: true + + - name: Upload validation reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: telemetry-validation-reports + path: /tmp/xrpld-validation/reports/ + retention-days: 30 + + - name: Upload node logs + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: xrpld-node-logs + path: /tmp/xrpld-validation/node*/debug.log + retention-days: 7 + + - name: Print validation summary + if: always() + run: | + REPORT="/tmp/xrpld-validation/reports/validation-report.json" + if [ -f "$REPORT" ]; then + echo "## Telemetry Validation Results" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + TOTAL=$(jq '.summary.total' "$REPORT") + PASSED=$(jq '.summary.passed' "$REPORT") + FAILED=$(jq '.summary.failed' "$REPORT") + echo "| Metric | Value |" >>"$GITHUB_STEP_SUMMARY" + echo "|--------|-------|" >>"$GITHUB_STEP_SUMMARY" + echo "| Total Checks | $TOTAL |" >>"$GITHUB_STEP_SUMMARY" + echo "| Passed | $PASSED |" >>"$GITHUB_STEP_SUMMARY" + echo "| Failed | $FAILED |" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + if [ "$FAILED" -gt 0 ]; then + echo "### Failed Checks" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + jq -r '.checks[] | select(.passed == false) | "- **\(.name)**: \(.message)"' "$REPORT" >>"$GITHUB_STEP_SUMMARY" + fi + fi + + # Publishes captured OTel timings + regression report to the Step Summary. + # When the committed baseline is a placeholder, emits a fenced JSON block + # that can be copy-pasted directly into baselines/baseline-timings.json. + # When the baseline is populated, summarises the top regressions so the + # PR author sees the failure reason without downloading artifacts. + - name: Print regression summary + if: always() + run: | + set -euo pipefail + TIMINGS="/tmp/xrpld-validation/reports/timings.json" + REGRESSION="/tmp/xrpld-validation/reports/regression-report.json" + BASELINE="docker/telemetry/workload/baselines/baseline-timings.json" + + if [ ! -f "$TIMINGS" ]; then + echo "## Regression Gate: no timings captured" >>"$GITHUB_STEP_SUMMARY" + echo "::warning::capture_timings.py did not produce timings.json — regression gate was not evaluated." + exit 0 + fi + + if [ ! -f "$BASELINE" ]; then + echo "## Regression Gate: baseline file missing" >>"$GITHUB_STEP_SUMMARY" + echo "::error::baselines/baseline-timings.json not found in checkout" + exit 1 + fi + + # NOTE: do NOT use `jq -e` here. With -e, jq exits non-zero when the + # filter's result is boolean false — which is the normal case for a + # populated (non-placeholder) baseline — and that would be + # misreported as a parse failure. Plain `jq -r` exits 0 on any valid + # JSON, so a real non-zero exit genuinely means malformed JSON. + IS_PLACEHOLDER=$(jq -r '.placeholder == true or (.metrics | length == 0)' "$BASELINE") || { + echo "::error::Failed to parse baseline JSON" + exit 1 + } + + echo "## OTel Timings Regression Gate" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + + if [ "$IS_PLACEHOLDER" = "true" ]; then + echo "### Paste into \`baselines/baseline-timings.json\`" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + echo "The committed baseline is a placeholder. Open a PR replacing" \ + "its contents with the JSON block below to activate the" \ + "regression gate." >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + echo '```json' >>"$GITHUB_STEP_SUMMARY" + cat "$TIMINGS" >>"$GITHUB_STEP_SUMMARY" + echo '```' >>"$GITHUB_STEP_SUMMARY" + elif [ -f "$REGRESSION" ]; then + REGR_COUNT=$(jq -e '.summary.regressions' "$REGRESSION") || REGR_COUNT=0 + IMPR_COUNT=$(jq -e '.summary.improvements' "$REGRESSION") || IMPR_COUNT=0 + TOTAL=$(jq -e '.summary.total' "$REGRESSION") || TOTAL=0 + echo "| Stat | Count |" >>"$GITHUB_STEP_SUMMARY" + echo "|------|-------|" >>"$GITHUB_STEP_SUMMARY" + echo "| Metrics compared | $TOTAL |" >>"$GITHUB_STEP_SUMMARY" + echo "| Regressions | $REGR_COUNT |" >>"$GITHUB_STEP_SUMMARY" + echo "| Improvements | $IMPR_COUNT |" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + if [ "$REGR_COUNT" -gt 0 ]; then + echo "### Regressions" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + echo "| Metric | Baseline | Current | Δ | % | Unit |" >>"$GITHUB_STEP_SUMMARY" + echo "|--------|---------:|--------:|--:|--:|------|" >>"$GITHUB_STEP_SUMMARY" + jq -r '.metrics[] | select(.regressed) | "| \(.key) | \(.baseline) | \(.current) | \(.delta) | \(.pct_change)% | \(.unit) |"' \ + "$REGRESSION" >>"$GITHUB_STEP_SUMMARY" + fi + fi + + - name: Cleanup + if: always() + run: | + docker/telemetry/workload/run-full-validation.sh --cleanup 2>/dev/null || true + + - name: Check validation result + if: steps.validation.outcome == 'failure' + run: | + echo "Telemetry validation failed. Check the uploaded reports for details." + exit 1 diff --git a/.gitignore b/.gitignore index 6bd34ece04c..2554435bf17 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,5 @@ __pycache__ # clangd cache /.cache +docker/telemetry/workload/__pycache__/ +.claude/ diff --git a/CMakeLists.txt b/CMakeLists.txt index f2e8fb3ae52..228bea4adc0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,6 +121,18 @@ if(rocksdb) target_link_libraries(xrpl_libs INTERFACE RocksDB::rocksdb) endif() +# OpenTelemetry distributed tracing (optional). +# When ON, links against opentelemetry-cpp and defines XRPL_ENABLE_TELEMETRY +# so that SpanGuard factory methods produce real OTel spans. +# When OFF (default), all tracing code compiles to no-ops with zero overhead. +# Enable via: conan install -o telemetry=True, or cmake -Dtelemetry=ON. +option(telemetry "Enable OpenTelemetry tracing" ON) +if(telemetry) + find_package(opentelemetry-cpp CONFIG REQUIRED) + add_compile_definitions(XRPL_ENABLE_TELEMETRY) + message(STATUS "OpenTelemetry tracing enabled") +endif() + # Work around changes to Conan recipe for now. if(TARGET nudb::core) set(nudb nudb::core) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7632741e358..d15548e239a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -354,6 +354,74 @@ If you wish to automatically fix whatever clang-tidy finds _and_ is capable of f run-clang-tidy -p build -quiet -fix -allow-no-checks src tests ``` +## Telemetry span attribute naming + +OpenTelemetry span attribute keys follow these rules so they stay consistent +across the code, the OTel collector, Tempo, Grafana dashboards, and docs. The +constants in the `*SpanNames.h` headers are the single source of truth; every +other layer must match them. A CI check enforces this end to end. + +1. Per-span unique attribute: bare field name — allowed when the field is + recorded by a single span/workflow, so the span name already supplies the + domain (e.g. `command`, `local`, `version` on `rpc.command` / `tx.process`). +2. Shared attribute (same concept on more than one span): ONE key, reused + verbatim on every span that records it — the span name tells the occurrences + apart, so no per-emitter prefix is added. Pick the name by the field's + meaning: a property of a domain object keeps that object's bare field name + (`ledger_hash`, `ledger_seq`, `tx_hash`, `peer_id`, `full_validation`); a + field already qualified by a sub-kind keeps that qualifier on every emitter + (`proposal_trusted` on both `consensus.proposal.receive` and + `peer.proposal.receive`; `validation_trusted` likewise). Define it once in + the base `SpanNames.h` `namespace attr` block and re-export (`using`) it from + each domain header, so all emitters share the exact string. +3. Collision qualifier: `_` — only when a bare name would collide + with a DIFFERENT concept in the shared spanmetrics label space, or with the + OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`, + `consensus_phase`, `consensus_round`). This disambiguates distinct concepts + that share a word; it is NOT used to tag the same concept with the workflow + that emitted it — that is rule 2 (one shared name). +4. Resource attribute: dotted `xrpl..` — reserved ONLY for + process/network identity set once at startup (`xrpl.network.id`, + `xrpl.network.type`). Never use the dotted `xrpl.` form for span attributes. +5. Span names use `[.]` (dotted). Only attribute _keys_ + follow rules 1–4. + +All attribute keys are `lower_snake_case` (lowercase letters, digits, and +underscores; each dot-separated segment of a resource key likewise). No +camelCase, uppercase, or spaces. + +Standard OpenTelemetry semantic-convention keys keep their canonical dotted +form (e.g. `service.*` resource attributes, `http.*` span attributes); the +"no dotted form" rule above applies to xrpl-custom keys, not to OTel-standard +conventions. + +Always reference the `*SpanNames.h` constants for attribute keys and span +names — never pass a string literal as a key or as a `span`/`childSpan` name +argument. (Attribute _values_ may be runtime data.) + +These rules are enforced by `.github/scripts/otel-naming/check_otel_naming.py`, +run in CI on every pull request. The check derives the set of valid keys +directly from the `*SpanNames.h` constants and the resource attributes the code +registers, so there is no separate list to keep in sync. It cross-validates the +collector, Tempo, dashboards, and docs against those keys, and each rule runs +only when the file it needs is present — so it works whether telemetry changes +land in one pull request or several. Run it locally with: + +``` +python .github/scripts/otel-naming/check_otel_naming.py +``` + +See [.github/scripts/otel-naming/README.md](.github/scripts/otel-naming/README.md) +for the full rule list. + +## Adding a new OTel metric + +See `src/xrpld/telemetry/MetricMacros.h` for the call-site macros covering every +OTel instrument kind (Counter, UpDownCounter, Histogram, Gauge, and their +Observable/async counterparts) and the "Adding a New Metric" section in +[docs/telemetry-runbook.md](docs/telemetry-runbook.md) for the walkthrough and a +need-to-macro lookup table. + ## Contracts and instrumentation We are using [Antithesis](https://antithesis.com/) for continuous fuzzing, diff --git a/OpenTelemetryPlan/00-tracing-fundamentals.md b/OpenTelemetryPlan/00-tracing-fundamentals.md new file mode 100644 index 00000000000..9c6f96d7afd --- /dev/null +++ b/OpenTelemetryPlan/00-tracing-fundamentals.md @@ -0,0 +1,565 @@ +# Distributed Tracing Fundamentals + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Next**: [Architecture Analysis](./01-architecture-analysis.md) + +--- + +## What is Distributed Tracing? + +Distributed tracing is a method for tracking data objects as they flow through distributed systems. In a network like XRP Ledger, a single transaction touches multiple independent nodes—each with no shared memory or logging. Distributed tracing connects these dots. + +**Without tracing:** You see isolated logs on each node with no way to correlate them. + +**With tracing:** You see the complete journey of a transaction or an event across all nodes it touched. + +--- + +## Actors and Actions at a Glance + +### Actors + +| Who (Plain English) | Technical Term | +| ---------------------------------------------- | --------------- | +| A single unit of work being tracked | Span | +| The complete journey of a request | Trace | +| Data that links spans across services | Trace Context | +| Code that creates spans and propagates context | Instrumentation | +| Service that receives and processes traces | Collector | +| Storage and visualization system | Backend (Tempo) | +| Decision logic for which traces to keep | Sampler | + +### Actions + +| What Happens (Plain English) | Technical Term | +| --------------------------------------- | ----------------------- | +| Start tracking a new operation | Create a Span | +| Connect a child operation to its parent | Set `parent_span_id` | +| Group all related operations together | Share a `trace_id` | +| Pass tracking data between services | Context Propagation | +| Decide whether to record a trace | Sampling (Head or Tail) | +| Send completed traces to storage | Export (OTLP) | + +--- + +## Core Concepts + +### 1. Trace + +A **trace** represents the entire journey of a request through the system. It has a unique `trace_id` that stays constant across all nodes. + +``` +Trace ID: abc123 +├── Node A: received transaction +├── Node B: relayed transaction +├── Node C: included in consensus +└── Node D: applied to ledger +``` + +### 2. Span + +A **span** represents a single unit of work within a trace. Each span has: + +| Attribute | Description | Example | +| ---------------- | -------------------------------- | -------------------------- | +| `trace_id` | Identifies the trace | `event123` | +| `span_id` | Unique identifier | `span456` | +| `parent_span_id` | Parent span (if any) | `p_span123` | +| `name` | Operation name | `rpc.submit` | +| `start_time` | When work began (local time) | `2024-01-15T10:30:00Z` | +| `end_time` | When work completed (local time) | `2024-01-15T10:30:00.050Z` | +| `attributes` | Key-value metadata | `tx_hash=ABC...` | +| `status` | OK, ERROR MSG | `OK` | + +### 3. Trace Context + +**Trace context** is the data that propagates between services to link spans together. It contains: + +- `trace_id` - The trace this span belongs to +- `span_id` - The current span (becomes parent for child spans) +- `trace_flags` - Sampling decisions + +--- + +## How Spans Form a Trace + +Spans have parent-child relationships forming a tree structure: + +```mermaid +flowchart TB + subgraph trace["Trace: abc123"] + A["tx.submit
span_id: 001
50ms"] --> B["tx.validate
span_id: 002
5ms"] + A --> C["tx.relay
span_id: 003
10ms"] + A --> D["tx.apply
span_id: 004
30ms"] + D --> E["ledger.update
span_id: 005
20ms"] + end + + style A fill:#0d47a1,stroke:#082f6a,color:#ffffff + style B fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style C fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style D fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style E fill:#bf360c,stroke:#8c2809,color:#ffffff +``` + +**Reading the diagram:** + +- **tx.submit (blue, root)**: The top-level span representing the entire transaction submission; all other spans are its descendants. +- **tx.validate, tx.relay, tx.apply (green)**: Direct children of tx.submit, representing the three main stages -- validation, relay to peers, and application to the ledger. +- **ledger.update (red)**: A grandchild span nested under tx.apply, representing the actual ledger state mutation triggered by applying the transaction. +- **Arrows (parent to child)**: Each arrow indicates a parent-child span relationship where the parent's completion depends on the child finishing. + +The same trace visualized as a **timeline (Gantt chart)**: + +``` +Time → 0ms 10ms 20ms 30ms 40ms 50ms + ├───────────────────────────────────────────┤ +tx.submit│▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓│ + ├─────┤ +tx.valid │▓▓▓▓▓│ + │ ├──────────┤ +tx.relay │ │▓▓▓▓▓▓▓▓▓▓│ + │ ├────────────────────────────┤ +tx.apply │ │▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓│ + │ ├──────────────────┤ +ledger │ │▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓│ +``` + +--- + +## Span Relationships + +Spans don't always form simple parent-child trees. Distributed tracing defines several relationship types to capture different causal patterns: + +### 1. Parent-Child (ChildOf) + +The default relationship. The parent span **depends on** or **contains** the child span. The child runs within the scope of the parent. + +``` +tx.submit (parent) +├── tx.validate (child) ← parent waits for this +├── tx.relay (child) ← parent waits for this +└── tx.apply (child) ← parent waits for this +``` + +**When to use:** Synchronous calls, nested operations, any case where the parent's completion depends on the child. + +### 2. Follows-From + +A causal relationship where the first span **triggers** the second, but does **not wait** for it. The originator fires and moves on. + +``` +Time → + +tx.receive [=======] + ↓ triggers (follows-from) + tx.relay [===========] ← runs independently +``` + +**When to use:** Asynchronous jobs, queued work, fire-and-forget patterns. For example, a node receives a transaction and queues it for relay — the relay span _follows from_ the receive span but the receiver doesn't wait for relaying to complete. + +> **OpenTracing** defined `FollowsFrom` as a first-class reference type alongside `ChildOf`. +> **OpenTelemetry** represents this using **Span Links** with descriptive attributes instead (see below). + +### 3. Span Links (Cross-Trace and Non-Hierarchical) + +Links connect spans that are **causally related but not in a parent-child hierarchy**. Unlike parent-child, links can cross trace boundaries. + +``` +Trace A Trace B +────── ────── +batch.schedule batch.execute +├─ item.enqueue (span X) ┌──► process.item +├─ item.enqueue (span Y) ───┤ (links to X, Y, Z) +├─ item.enqueue (span Z) └──► +``` + +**Use cases:** + +| Pattern | Description | +| -------------------- | --------------------------------------------------------------------------- | +| **Batch processing** | A batch span links back to all individual spans that contributed to it | +| **Fan-in** | An aggregation span links to the multiple producer spans it merges | +| **Fan-out** | Multiple downstream spans link back to the single span that triggered them | +| **Async handoff** | A deferred job links back to the request that queued it (follows-from) | +| **Cross-trace** | Correlating spans across independent traces (e.g., retries, related events) | + +**Link structure:** Each link carries the target span's context plus optional attributes: + +``` +Link { + trace_id: + span_id: + attributes: { "link.description": "triggered by batch scheduler" } +} +``` + +### Relationship Summary + +```mermaid +flowchart LR + subgraph parent_child["Parent-Child"] + direction TB + P["Parent"] --> C["Child"] + end + + subgraph follows_from["Follows-From"] + direction TB + A["Span A"] -.->|triggers| B["Span B"] + end + + subgraph links["Span Links"] + direction TB + X["Span X\n(Trace 1)"] -.-|link| Y["Span Y\n(Trace 2)"] + end + + parent_child ~~~ follows_from ~~~ links + + style P fill:#0d47a1,stroke:#082f6a,color:#ffffff + style C fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style A fill:#0d47a1,stroke:#082f6a,color:#ffffff + style B fill:#bf360c,stroke:#8c2809,color:#ffffff + style X fill:#4a148c,stroke:#38006b,color:#ffffff + style Y fill:#4a148c,stroke:#38006b,color:#ffffff +``` + +| Relationship | Same Trace? | Dependency? | OTel Mechanism | +| ---------------- | ----------- | -------------------------- | ----------------- | +| **Parent-Child** | Yes | Parent depends on child | `parent_span_id` | +| **Follows-From** | Usually | Causal but no dependency | Link + attributes | +| **Span Link** | Either | Correlation, no dependency | Link + attributes | + +--- + +## Trace ID Generation + +A `trace_id` is a 128-bit (16-byte) identifier that groups all spans belonging to one logical operation. How it's generated determines how easily you can find and correlate traces later. + +### General Approaches + +#### 1. Random (W3C Default) + +Generate a random 128-bit ID when a trace starts. Standard approach for most services. + +``` +trace_id = random_128_bits() +``` + +| Pros | Cons | +| --------------------------- | --------------------------------------------- | +| Simple, standard | No natural correlation to domain events | +| Guaranteed unique per trace | If propagation is lost, trace is broken | +| Works with all OTel tooling | "Find trace for TX abc" requires index lookup | + +#### 2. Deterministic (Derived from Domain Data) + +Compute the trace_id from a hash of a natural identifier. Every node independently derives the **same** trace_id for the same event. + +``` +trace_id = SHA-256(domain_identifier)[0:16] // truncate to 128 bits +``` + +| Pros | Cons | +| --------------------------------------------------- | ---------------------------------------------------------- | +| Propagation-resilient — same ID computed everywhere | Same event processed twice (retry) shares trace_id | +| Natural search — domain ID maps directly to trace | Non-standard (tooling assumes random) | +| No coordination needed between nodes | 256→128 bit truncation (collision risk negligible at ~2⁶⁴) | + +#### 3. Hybrid (Deterministic Prefix + Random Suffix) + +First 8 bytes derived from domain data, last 8 bytes random. + +``` +trace_id = SHA-256(domain_identifier)[0:8] || random_64_bits() +``` + +| Pros | Cons | +| ------------------------------------------- | ---------------------------------------- | +| Prefix search: "find all traces for TX abc" | Must propagate to maintain full trace_id | +| Unique per processing instance | More complex generation logic | +| Retries get distinct trace_ids | Partial correlation only (prefix match) | + +### XRPL Workflow Analysis + +XRPL has a unique advantage: its core workflows produce **globally unique 256-bit hashes** that are known on every node. This makes deterministic trace_id generation practical in ways most systems can't achieve. + +#### Natural Identifiers by Workflow + +| Workflow | Natural Identifier | Size | Known at Start? | Same on All Nodes? | +| ------------------- | --------------------------------- | ---------- | ----------------------------- | -------------------------------- | +| **Transaction** | Transaction hash (`tid_`) | 256-bit | Yes — computed before signing | Yes — hash of canonical tx data | +| **Consensus round** | Previous ledger hash + ledger seq | 256+32 bit | Yes — known when round opens | Yes — all validators agree | +| **Validation** | Ledger hash being validated | 256-bit | Yes — from consensus result | Yes — same closed ledger | +| **Ledger catch-up** | Target ledger hash | 256-bit | Yes — we know what to fetch | Yes — identifies ledger globally | + +#### Where These Identifiers Live in Code + +``` +Transaction: STTx::getTransactionID() → uint256 tid_ + TMTransaction::rawTransaction → recompute hash from bytes + +Consensus: ConsensusProposal::prevLedger_ → uint256 (previous ledger hash) + ConsensusProposal::position_ → uint256 (TxSet hash) + LedgerHeader::seq → uint32_t (ledger sequence) + +Validation: STValidation::getLedgerHash() → uint256 + STValidation::getNodeID() → NodeID (160-bit) + +Ledger fetch: InboundLedger constructor → uint256 hash, uint32_t seq + TMGetLedger::ledgerHash → bytes (uint256) +``` + +### Recommended Strategy: Workflow-Scoped Deterministic + +Each workflow type derives its trace_id from its natural domain identifier: + +``` +Transaction trace: trace_id = SHA-256("tx" || tx_hash)[0:16] +Consensus trace: trace_id = SHA-256("cons" || prev_ledger_hash || ledger_seq)[0:16] +Ledger catch-up: trace_id = SHA-256("fetch" || target_ledger_hash)[0:16] +``` + +The string prefix (`"tx"`, `"cons"`, `"fetch"`) prevents collisions between workflows that might share underlying hashes. + +**Why this works for XRPL:** + +1. **Propagation-resilient** — Even if a P2P message drops trace context, every node independently computes the same trace_id from the same tx_hash or ledger_hash. Spans still correlate. + +2. **Zero-cost search** — "Show me the trace for transaction ABC" becomes a direct lookup: compute `SHA-256("tx" || ABC)[0:16]` and query. No secondary index needed. + +3. **Cross-workflow linking via Span Links** — A consensus trace links to individual transaction traces. A validation span links to the consensus trace. This connects the full picture without forcing everything into one giant trace. + +### Cross-Workflow Correlation + +Each workflow gets its own trace. Span Links tie them together: + +```mermaid +flowchart TB + subgraph tx_trace["Transaction Trace"] + direction LR + Tn["trace_id = f(tx_hash)"]:::note --> T1["tx.receive"] --> T2["tx.validate"] --> T3["tx.relay"] + end + + subgraph cons_trace["Consensus Trace"] + direction LR + Cn["trace_id = f(prev_ledger, seq)"]:::note --> C1["cons.open"] --> C2["cons.propose"] --> C3["cons.accept"] + end + + subgraph val_trace["Validation"] + direction LR + Vn["spans within consensus trace"]:::note --> V1["val.create"] --> V2["val.broadcast"] + end + + subgraph fetch_trace["Catch-Up Trace"] + direction LR + Fn["trace_id = f(ledger_hash)"]:::note --> F1["fetch.request"] --> F2["fetch.receive"] --> F3["fetch.apply"] + end + + C1 -.-|"span link\n(tx traces)"| T3 + C3 --> V1 + F1 -.-|"span link\n(target ledger)"| C3 + + classDef note fill:none,stroke:#888,stroke-dasharray:5 5,color:#333,font-style:italic + style T1 fill:#0d47a1,stroke:#082f6a,color:#ffffff + style T2 fill:#0d47a1,stroke:#082f6a,color:#ffffff + style T3 fill:#0d47a1,stroke:#082f6a,color:#ffffff + style C1 fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style C2 fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style C3 fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style V1 fill:#bf360c,stroke:#8c2809,color:#ffffff + style V2 fill:#bf360c,stroke:#8c2809,color:#ffffff + style F1 fill:#4a148c,stroke:#38006b,color:#ffffff + style F2 fill:#4a148c,stroke:#38006b,color:#ffffff + style F3 fill:#4a148c,stroke:#38006b,color:#ffffff +``` + +**Reading the diagram:** + +- **Transaction Trace (blue)**: An independent trace whose `trace_id` is deterministically derived from the transaction hash. Contains receive, validate, and relay spans. +- **Consensus Trace (green)**: An independent trace whose `trace_id` is derived from the previous ledger hash and sequence number. Covers the open, propose, and accept phases. +- **Validation (red)**: Validation spans live within the consensus trace (not a separate trace). They are created after the accept phase completes. +- **Catch-Up Trace (purple)**: An independent trace for ledger acquisition, derived from the target ledger hash. Used when a node is behind and fetching missing ledgers. +- **Dotted arrows (span links)**: Cross-trace correlations. Consensus links to transaction traces it included; catch-up links to the consensus trace that produced the target ledger. +- **Solid arrow (C3 to V1)**: A parent-child relationship -- validation spans are direct children of the consensus accept span within the same trace. + +**How a query flows:** + +``` +"Why was TX abc slow?" + 1. Compute trace_id = SHA-256("tx" || abc)[0:16] + 2. Find transaction trace → see it was included in consensus round N + 3. Follow span link → consensus trace for round N + 4. See which phase was slow (propose? accept?) + 5. If a node was catching up, follow link → catch-up trace +``` + +### Trade-offs to Consider + +| Concern | Mitigation | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| **Retries get same trace_id** | Add `attempt` attribute to root span; spans have unique span_ids and timestamps | +| **256→128 bit truncation** | Birthday-bound collision at ~2⁶⁴ operations — negligible for XRPL's throughput | +| **Non-standard generation** | OTel spec allows any 16-byte non-zero value; tooling works on the hex string | +| **Hash computation cost** | SHA-256 is ~0.3μs per call; XRPL already computes these hashes for other purposes | +| **Late-binding identifiers** | Ledger hash isn't known until after consensus — validation spans use ledger_seq as fallback, then link to the consensus trace | + +--- + +## Distributed Traces Across Nodes + +In distributed systems like xrpld, traces span **multiple independent nodes**. The trace context must be propagated in network messages: + +```mermaid +sequenceDiagram + participant Client + participant NodeA as Node A + participant NodeB as Node B + participant NodeC as Node C + + Client->>NodeA: Submit TX
(no trace context) + + Note over NodeA: Creates new trace
trace_id: abc123
span: tx.receive + + NodeA->>NodeB: Relay TX
(trace_id: abc123, parent: 001) + + Note over NodeB: Creates child span
span: tx.relay
parent_span_id: 001 + + NodeA->>NodeC: Relay TX
(trace_id: abc123, parent: 001) + + Note over NodeC: Creates child span
span: tx.relay
parent_span_id: 001 + + Note over NodeA,NodeC: All spans share trace_id: abc123
enabling correlation across nodes +``` + +**Reading the diagram:** + +- **Client**: The external entity that submits a transaction. It does not carry trace context -- the trace originates at the first node. +- **Node A**: The entry point that creates a new trace (trace_id: abc123) and the root span `tx.receive`. It relays the transaction to peers with trace context attached. +- **Node B and Node C**: Peer nodes that receive the relayed transaction along with the propagated trace context. Each creates a child span under Node A's span, preserving the same `trace_id`. +- **Arrows with trace context**: The relay messages carry `trace_id` and `parent_span_id`, allowing each downstream node to link its spans back to the originating span on Node A. + +--- + +## Context Propagation + +For traces to work across nodes, **trace context must be propagated** in messages. + +### What's in the Context (~26 bytes) + +| Field | Size | Description | +| ------------- | -------- | ------------------------------------------------------- | +| `trace_id` | 16 bytes | Identifies the entire trace (constant across all nodes) | +| `span_id` | 8 bytes | The sender's current span (becomes parent on receiver) | +| `trace_flags` | 1 byte | Sampling decision (bit 0 = sampled; bits 1-7 reserved) | +| `trace_state` | variable | Optional vendor-specific data (typically omitted) | + +### How span_id Changes at Each Hop + +Only **one** `span_id` travels in the context - the sender's current span. Each node: + +1. Extracts the received `span_id` and uses it as the `parent_span_id` +2. Creates a **new** `span_id` for its own span +3. Sends its own `span_id` as the parent when forwarding + +``` +Node A Node B Node C +────── ────── ────── + +Span AAA Span BBB Span CCC + │ │ │ + ▼ ▼ ▼ +Context out: Context out: Context out: +├─ trace_id: abc123 ├─ trace_id: abc123 ├─ trace_id: abc123 +├─ span_id: AAA ──────────► ├─ span_id: BBB ──────────► ├─ span_id: CCC ──────► +└─ flags: 01 └─ flags: 01 └─ flags: 01 + │ │ + parent = AAA parent = BBB +``` + +The `trace_id` stays constant, but `span_id` **changes at every hop** to maintain the parent-child chain. + +### Propagation Formats + +There are two patterns: + +### HTTP/RPC Headers (W3C Trace Context) + +``` +traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01 + │ │ │ │ + │ │ │ └── Flags (sampled) + │ │ └── Parent span ID (16 hex) + │ └── Trace ID (32 hex) + └── Version +``` + +### Protocol Buffers (xrpld P2P messages) + +xrpld P2P messages such as `TMTransaction` carry the trace context in two added byte fields alongside the existing payload: `trace_parent` holds the W3C traceparent (`trace_id`, `span_id`, and `trace_flags`), and `trace_state` holds the optional W3C tracestate. Together they propagate the trace across the P2P boundary so a receiving node can attach its spans to the sender's span. + +--- + +## Sampling + +Not every trace needs to be recorded. **Sampling** reduces overhead: + +### Head Sampling (at trace start) + +``` +Request arrives → Random N% chance → Record or skip entire trace +``` + +- ✅ Low overhead +- ❌ May miss interesting traces + +> **xrpld note**: xrpld intentionally fixes head sampling at 100% (sample +> everything) and does not expose a configurable ratio. A per-node ratio +> would let different nodes make divergent keep/drop decisions for the same +> distributed trace, producing broken/partial traces. xrpld uses a +> `ParentBased` sampler so spans with a remote parent honor the upstream +> decision. Volume reduction is delegated to collector-side tail sampling. + +### Tail Sampling (after trace completes) + +``` +Trace completes → Collector evaluates: + - Error? → KEEP + - Slow? → KEEP + - Normal? → Sample 10% +``` + +- ✅ Never loses important traces +- ❌ Higher memory usage at collector + +--- + +## Key Benefits for xrpld + +| Challenge | How Tracing Helps | +| ---------------------------------- | ---------------------------------------- | +| "Where is my transaction?" | Follow trace across all nodes it touched | +| "Why was consensus slow?" | See timing breakdown of each phase | +| "Which node is the bottleneck?" | Compare span durations across nodes | +| "What happened during the outage?" | Correlate errors across the network | + +--- + +## Glossary + +| Term | Definition | +| -------------------- | ------------------------------------------------------------------- | +| **Trace** | Complete journey of a request, identified by `trace_id` | +| **Span** | Single operation within a trace | +| **Parent-Child** | Span relationship where the parent depends on the child | +| **Follows-From** | Causal relationship where originator doesn't wait for the result | +| **Span Link** | Non-hierarchical connection between spans, possibly across traces | +| **Deterministic ID** | Trace ID derived from domain data (e.g., tx_hash) instead of random | +| **Context** | Data propagated between services (`trace_id`, `span_id`, flags) | +| **Instrumentation** | Code that creates spans and propagates context | +| **Collector** | Service that receives, processes, and exports traces | +| **Backend** | Storage/visualization system (Tempo) | +| **Head Sampling** | Sampling decision at trace start | +| **Tail Sampling** | Sampling decision after trace completes | + +--- + +_Next: [Architecture Analysis](./01-architecture-analysis.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/01-architecture-analysis.md b/OpenTelemetryPlan/01-architecture-analysis.md new file mode 100644 index 00000000000..1161a1645b0 --- /dev/null +++ b/OpenTelemetryPlan/01-architecture-analysis.md @@ -0,0 +1,467 @@ +# Architecture Analysis + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Design Decisions](./02-design-decisions.md) | [Implementation Strategy](./03-implementation-strategy.md) + +--- + +## 1.1 Current xrpld Architecture Overview + +> **WS** = WebSocket | **UNL** = Unique Node List | **TxQ** = Transaction Queue | **StatsD** = Statistics Daemon + +The xrpld node software consists of several interconnected components that need instrumentation for distributed tracing: + +```mermaid +flowchart TB + subgraph xrpld["xrpld Node"] + subgraph services["Core Services"] + RPC["RPC Server
(HTTP/WS/gRPC)"] + Overlay["Overlay
(P2P Network)"] + Consensus["Consensus
(RCLConsensus)"] + ValidatorList["ValidatorList
(UNL Mgmt)"] + end + + JobQueue["JobQueue
(Thread Pool)"] + + subgraph processing["Processing Layer"] + NetworkOPs["NetworkOPs
(Tx Processing)"] + LedgerMaster["LedgerMaster
(Ledger Mgmt)"] + NodeStore["NodeStore
(Database)"] + InboundLedgers["InboundLedgers
(Ledger Sync)"] + end + + subgraph appservices["Application Services"] + PathFind["PathFinding
(Payment Paths)"] + TxQ["TxQ
(Fee Escalation)"] + LoadMgr["LoadManager
(Fee/Load)"] + end + + subgraph observability["Existing Observability"] + PerfLog["PerfLog
(JSON)"] + Insight["Insight
(StatsD)"] + Logging["Logging
(Journal)"] + end + + services --> JobQueue + JobQueue --> processing + JobQueue --> appservices + end + + style xrpld fill:#424242,stroke:#212121,color:#ffffff + style services fill:#1565c0,stroke:#0d47a1,color:#ffffff + style processing fill:#2e7d32,stroke:#1b5e20,color:#ffffff + style appservices fill:#6a1b9a,stroke:#4a148c,color:#ffffff + style observability fill:#e65100,stroke:#bf360c,color:#ffffff +``` + +**Reading the diagram:** + +- **Core Services (blue)**: The entry points into xrpld -- RPC Server handles client requests, Overlay manages peer-to-peer networking, Consensus drives agreement, and ValidatorList manages trusted validators. +- **JobQueue (center)**: The asynchronous thread pool that decouples Core Services from the Processing and Application layers. All work flows through it. +- **Processing Layer (green)**: Core business logic -- NetworkOPs processes transactions, LedgerMaster manages ledger state, NodeStore handles persistence, and InboundLedgers synchronizes missing data. +- **Application Services (purple)**: Higher-level features -- PathFinding computes payment routes, TxQ manages fee-based queuing, and LoadManager tracks server load. +- **Existing Observability (orange)**: The current monitoring stack (PerfLog, Insight, Journal logging) that OpenTelemetry will complement, not replace. +- **Arrows (Services to JobQueue to layers)**: Work originates at Core Services, is enqueued onto the JobQueue, and dispatched to Processing or Application layers for execution. + +--- + +## 1.1.1 Actors and Actions + +### Actors + +| Who (Plain English) | Technical Term | +| ----------------------------------------- | -------------------------- | +| Network node running XRPL software | xrpld node | +| External client submitting requests | RPC Client | +| Network neighbor sharing data | Peer (PeerImp) | +| Request handler for client queries | RPC Server (ServerHandler) | +| Command executor for specific RPC methods | RPCHandler | +| Agreement process between nodes | Consensus (RCLConsensus) | +| Transaction processing coordinator | NetworkOPs | +| Background task scheduler | JobQueue | +| Ledger state manager | LedgerMaster | +| Payment route calculator | PathFinding (Pathfinder) | +| Transaction waiting room | TxQ (Transaction Queue) | +| Fee adjustment system | LoadManager | +| Trusted validator list manager | ValidatorList | +| Protocol upgrade tracker | AmendmentTable | +| Ledger state hash tree | SHAMap | +| Persistent key-value storage | NodeStore | + +### Actions + +| What Happens (Plain English) | Technical Term | +| ---------------------------------------------- | ---------------------- | +| Client sends a request to a node | `rpc.request` | +| Node executes a specific RPC command | `rpc.command.*` | +| Node receives a transaction from a peer | `tx.receive` | +| Node checks if a transaction is valid | `tx.validate` | +| Node forwards a transaction to neighbors | `tx.relay` | +| Nodes agree on which transactions to include | `consensus.round` | +| Consensus progresses through phases | `consensus.phase.*` | +| Node builds a new confirmed ledger | `ledger.build` | +| Node fetches missing ledger data from peers | `ledger.acquire` | +| Node computes payment routes | `pathfind.compute` | +| Node queues a transaction for later processing | `txq.enqueue` | +| Node increases fees due to high load | `fee.escalate` | +| Node fetches the latest trusted validator list | `validator.list.fetch` | +| Node votes on a protocol amendment | `amendment.vote` | +| Node synchronizes state tree data | `shamap.sync` | + +--- + +## 1.2 Key Components for Instrumentation + +> **TxQ** = Transaction Queue | **UNL** = Unique Node List + +| Component | Location | Purpose | Trace Value | +| ------------------ | ------------------------------------------ | ------------------------ | -------------------------------- | +| **Overlay** | `src/xrpld/overlay/` | P2P communication | Message propagation timing | +| **PeerImp** | `src/xrpld/overlay/detail/PeerImp.cpp` | Individual peer handling | Per-peer latency | +| **RCLConsensus** | `src/xrpld/app/consensus/RCLConsensus.cpp` | Consensus algorithm | Round timing, phase analysis | +| **NetworkOPs** | `src/xrpld/app/misc/NetworkOPs.cpp` | Transaction processing | Tx lifecycle tracking | +| **ServerHandler** | `src/xrpld/rpc/detail/ServerHandler.cpp` | RPC entry point | Request latency | +| **RPCHandler** | `src/xrpld/rpc/detail/RPCHandler.cpp` | Command execution | Per-command timing | +| **JobQueue** | `src/xrpl/core/JobQueue.h` | Async task execution | Queue wait times | +| **PathFinding** | `src/xrpld/app/paths/` | Payment path computation | Path latency, cache hits | +| **TxQ** | `src/xrpld/app/misc/TxQ.cpp` | Transaction queue/fees | Queue depth, eviction rates | +| **LoadManager** | `src/xrpld/app/main/LoadManager.cpp` | Fee escalation/load | Fee levels, load factors | +| **InboundLedgers** | `src/xrpld/app/ledger/InboundLedgers.cpp` | Ledger acquisition | Sync time, peer reliability | +| **ValidatorList** | `src/xrpld/app/misc/ValidatorList.cpp` | UNL management | List freshness, fetch failures | +| **AmendmentTable** | `src/xrpld/app/misc/AmendmentTable.cpp` | Protocol amendments | Voting status, activation events | +| **SHAMap** | `src/xrpld/shamap/` | State hash tree | Sync speed, missing nodes | + +--- + +## 1.3 Transaction Flow Diagram + +Transaction flow spans multiple nodes in the network. Each node creates linked spans to form a distributed trace: + +```mermaid +sequenceDiagram + participant Client + participant PeerA as Peer A (Receive) + participant PeerB as Peer B (Relay) + participant PeerC as Peer C (Validate) + + Client->>PeerA: 1. Submit TX + + rect rgb(230, 245, 255) + Note over PeerA: tx.receive SPAN START + PeerA->>PeerA: HashRouter Deduplication + PeerA->>PeerA: tx.validate (child span) + end + + PeerA->>PeerB: 2. Relay TX (with trace ctx) + + rect rgb(230, 245, 255) + Note over PeerB: tx.receive (linked span) + end + + PeerB->>PeerC: 3. Relay TX + + rect rgb(230, 245, 255) + Note over PeerC: tx.receive (linked span) + PeerC->>PeerC: tx.process + end + + Note over Client,PeerC: DISTRIBUTED TRACE (same trace_id: abc123) +``` + +**Reading the diagram:** + +- **Client**: The external entity that submits a transaction to Peer A. It has no trace context -- the trace starts at the first node. +- **Peer A (Receive)**: The entry node that creates the root span `tx.receive`, runs HashRouter deduplication to avoid processing duplicates, and creates a child `tx.validate` span. +- **Peer A to Peer B arrow**: The relay message carries trace context (trace_id + parent span_id), enabling Peer B to create a linked span under the same trace. +- **Peer B (Relay)**: Receives the transaction and trace context, creates a `tx.receive` span linked to Peer A's trace, then relays onward. +- **Peer C (Validate)**: Final hop in this example. Creates a linked `tx.receive` span and runs `tx.process` to fully process the transaction. +- **Blue rectangles**: Highlight the span boundaries on each node, showing where instrumentation creates and closes spans. + +### Trace Structure + +``` +trace_id: abc123 +├── span: tx.receive (Peer A) +│ ├── span: tx.validate +│ └── span: tx.relay +├── span: tx.receive (Peer B) [parent: Peer A] +│ └── span: tx.relay +└── span: tx.receive (Peer C) [parent: Peer B] + └── span: tx.process +``` + +--- + +## 1.4 Consensus Round Flow + +Consensus rounds are multi-phase operations that benefit significantly from tracing: + +```mermaid +flowchart TB + subgraph round["consensus.round (root span)"] + attrs["Attributes:
ledger_seq = 12345678
consensus_mode = proposing
proposers = 35"] + + subgraph open["consensus.phase.open"] + open_desc["Duration: ~3s
Waiting for transactions"] + end + + subgraph establish["consensus.phase.establish"] + est_attrs["proposals_received = 28
disputes_resolved = 3"] + est_children["├── consensus.proposal.receive (×28)
├── consensus.proposal.send (×1)
└── consensus.dispute.resolve (×3)"] + end + + subgraph accept["consensus.phase.accept"] + acc_attrs["transactions_applied = 150
ledger_hash = DEF456..."] + acc_children["├── ledger.build
└── ledger.validate"] + end + + attrs --> open + open --> establish + establish --> accept + end + + style round fill:#f57f17,stroke:#e65100,color:#ffffff + style open fill:#1565c0,stroke:#0d47a1,color:#ffffff + style establish fill:#2e7d32,stroke:#1b5e20,color:#ffffff + style accept fill:#c2185b,stroke:#880e4f,color:#ffffff +``` + +**Reading the diagram:** + +- **consensus.round (orange, root span)**: The top-level span encompassing the entire consensus round, with attributes like ledger sequence, mode, and proposer count. +- **consensus.phase.open (blue)**: The first phase where the node waits (~3s) to collect incoming transactions before proposing. +- **consensus.phase.establish (green)**: The negotiation phase where validators exchange proposals, resolve disputes, and converge on a transaction set. Child spans track each proposal received/sent and each dispute resolved. +- **consensus.phase.accept (pink)**: The final phase where the agreed transaction set is applied, a new ledger is built, and the ledger is validated. Child spans cover `ledger.build` and `ledger.validate`. +- **Arrows (open to establish to accept)**: The sequential flow through the three consensus phases. Each phase must complete before the next begins. + +--- + +## 1.5 RPC Request Flow + +> **WS** = WebSocket + +RPC requests support W3C Trace Context headers for distributed tracing across services: + +```mermaid +flowchart TB + subgraph request["rpc.request (root span)"] + http["HTTP Request — POST /
traceparent:
00-abc123...-def456...-01"] + + attrs["Attributes:
http.method = POST
net.peer.ip = 192.168.1.100
command = submit"] + + subgraph enqueue["jobqueue.enqueue"] + job_attr["job_type = jtCLIENT_RPC"] + end + + subgraph command["rpc.command.submit"] + cmd_attrs["version = 2
rpc_role = user"] + cmd_children["├── tx.deserialize
├── tx.validate_local
└── tx.submit_to_network"] + end + + response["Response: 200 OK
Duration: 45ms"] + + http --> attrs + attrs --> enqueue + enqueue --> command + command --> response + end + + style request fill:#2e7d32,stroke:#1b5e20,color:#ffffff + style enqueue fill:#1565c0,stroke:#0d47a1,color:#ffffff + style command fill:#e65100,stroke:#bf360c,color:#ffffff +``` + +**Reading the diagram:** + +- **rpc.request (green, root span)**: The outermost span representing the full RPC request lifecycle, from HTTP receipt to response. Carries the W3C `traceparent` header for distributed tracing. +- **HTTP Request node**: Shows the incoming POST request with its `traceparent` header and extracted attributes (method, peer IP, command name). +- **jobqueue.enqueue (blue)**: The span covering the asynchronous handoff from the RPC thread to the JobQueue worker thread. The trace context is preserved across this async boundary. +- **rpc.command.submit (orange)**: The span for the actual command execution, with child spans for deserialization, local validation, and network submission. +- **Response node**: The final output with HTTP status and total duration, marking the end of the root span. +- **Arrows (top to bottom)**: The sequential processing pipeline -- receive request, extract attributes, enqueue job, execute command, return response. + +--- + +## 1.6 Key Trace Points + +> **TxQ** = Transaction Queue + +The following table identifies priority instrumentation points across the codebase: + +| Category | Span Name | File | Method | Priority | +| --------------- | ---------------------- | ---------------------- | ----------------------- | -------- | +| **Transaction** | `tx.receive` | `PeerImp.cpp` | `handleTransaction()` | High | +| **Transaction** | `tx.validate` | `NetworkOPs.cpp` | `processTransaction()` | High | +| **Transaction** | `tx.process` | `NetworkOPs.cpp` | `doTransactionSync()` | High | +| **Transaction** | `tx.relay` | `OverlayImpl.cpp` | `relay()` | Medium | +| **Consensus** | `consensus.round` | `RCLConsensus.cpp` | `startRound()` | High | +| **Consensus** | `consensus.phase.*` | `Consensus.h` | `timerEntry()` | High | +| **Consensus** | `consensus.proposal.*` | `RCLConsensus.cpp` | `peerProposal()` | Medium | +| **RPC** | `rpc.request` | `ServerHandler.cpp` | `onRequest()` | High | +| **RPC** | `rpc.command.*` | `RPCHandler.cpp` | `doCommand()` | High | +| **Peer** | `peer.connect` | `OverlayImpl.cpp` | `onHandoff()` | Low | +| **Peer** | `peer.message.*` | `PeerImp.cpp` | `onMessage()` | Low | +| **Ledger** | `ledger.acquire` | `InboundLedgers.cpp` | `acquire()` | Medium | +| **Ledger** | `ledger.build` | `RCLConsensus.cpp` | `buildLCL()` | High | +| **PathFinding** | `pathfind.request` | `PathRequest.cpp` | `doUpdate()` | High | +| **PathFinding** | `pathfind.compute` | `Pathfinder.cpp` | `findPaths()` | High | +| **TxQ** | `txq.enqueue` | `TxQ.cpp` | `apply()` | High | +| **TxQ** | `txq.apply` | `TxQ.cpp` | `processClosedLedger()` | High | +| **Fee** | `fee.escalate` | `LoadManager.cpp` | `raiseLocalFee()` | Medium | +| **Ledger** | `ledger.replay` | `LedgerReplayer.h` | `replay()` | Medium | +| **Ledger** | `ledger.delta` | `LedgerDeltaAcquire.h` | `processData()` | Medium | +| **Validator** | `validator.list.fetch` | `ValidatorList.cpp` | `verify()` | Medium | +| **Validator** | `validator.manifest` | `Manifest.cpp` | `applyManifest()` | Low | +| **Amendment** | `amendment.vote` | `AmendmentTable.cpp` | `doVoting()` | Low | +| **SHAMap** | `shamap.sync` | `SHAMap.cpp` | `fetchRoot()` | Medium | + +--- + +## 1.7 Instrumentation Priority + +> **TxQ** = Transaction Queue + +```mermaid +quadrantChart + title Instrumentation Priority Matrix + x-axis Low Complexity --> High Complexity + y-axis Low Value --> High Value + quadrant-1 Implement First + quadrant-2 Plan Carefully + quadrant-3 Quick Wins + quadrant-4 Consider Later + + RPC Tracing: [0.2, 0.92] + Transaction Tracing: [0.55, 0.88] + Consensus Tracing: [0.78, 0.82] + PathFinding: [0.38, 0.75] + TxQ and Fees: [0.25, 0.65] + Ledger Sync: [0.62, 0.58] + Peer Message Tracing: [0.35, 0.25] + JobQueue Tracing: [0.2, 0.48] + Validator Mgmt: [0.48, 0.42] + Amendment Tracking: [0.15, 0.32] + SHAMap Operations: [0.72, 0.45] +``` + +--- + +## 1.8 Observable Outcomes + +> **TxQ** = Transaction Queue | **UNL** = Unique Node List + +After implementing OpenTelemetry, operators and developers will gain visibility into the following: + +### 1.8.1 What You Will See: Traces + +| Trace Type | Description | Example Query in Grafana/Tempo | +| -------------------------- | ------------------------------------------------------------------------------------------- | ----------------------------------------------- | +| **Transaction Lifecycle** | Full journey from RPC submission through validation, relay, consensus, and ledger inclusion | `{service.name="xrpld" && tx_hash="ABC123..."}` | +| **Cross-Node Propagation** | Transaction path across multiple xrpld nodes with timing | `{relay_count > 0}` | +| **Consensus Rounds** | Complete round with all phases (open, establish, accept) | `{span.name=~"consensus.round.*"}` | +| **RPC Request Processing** | Individual command execution with timing breakdown | `{command="account_info"}` | +| **Ledger Acquisition** | Peer-to-peer ledger data requests and responses | `{span.name="ledger.acquire"}` | +| **PathFinding Latency** | Path computation time and cache effectiveness for payment RPCs | `{span.name="pathfind.compute"}` | +| **TxQ Behavior** | Queue depth, eviction patterns, fee escalation during congestion | `{span.name=~"txq.*"}` | +| **Ledger Sync** | Full acquisition timeline including delta and transaction fetches | `{span.name=~"ledger.acquire.*"}` | +| **Validator Health** | UNL fetch success, manifest updates, stale list detection | `{span.name=~"validator.*"}` | + +### 1.8.2 What You Will See: Metrics (Derived from Traces) + +| Metric | Description | Dashboard Panel | +| ----------------------------- | --------------------------------------- | --------------------------- | +| **RPC Latency (p50/p95/p99)** | Response time distribution per command | Heatmap by command | +| **Transaction Throughput** | Transactions processed per second | Time series graph | +| **Consensus Round Duration** | Time to complete consensus phases | Histogram | +| **Cross-Node Latency** | Time for transaction to reach N nodes | Line chart with percentiles | +| **Error Rate** | Failed transactions/RPC calls by type | Stacked bar chart | +| **PathFinding Latency** | Path computation time per currency pair | Heatmap by currency | +| **TxQ Depth** | Queued transactions over time | Time series with thresholds | +| **Fee Escalation Level** | Current fee multiplier | Gauge with alert thresholds | +| **Ledger Sync Duration** | Time to acquire missing ledgers | Histogram | + +### 1.8.3 Concrete Dashboard Examples + +**Transaction Trace View (Tempo):** + +``` +┌────────────────────────────────────────────────────────────────────────────────┐ +│ Trace: abc123... (Transaction Submission) Duration: 847ms │ +├────────────────────────────────────────────────────────────────────────────────┤ +│ ├── rpc.request [ServerHandler] ████░░░░░░ 45ms │ +│ │ └── rpc.command.submit [RPCHandler] ████░░░░░░ 42ms │ +│ │ └── tx.receive [NetworkOPs] ███░░░░░░░ 35ms │ +│ │ ├── tx.validate [TxQ] █░░░░░░░░░ 8ms │ +│ │ └── tx.relay [Overlay] ██░░░░░░░░ 15ms │ +│ │ ├── tx.receive [Node-B] █████░░░░░ 52ms │ +│ │ │ └── tx.relay [Node-B] ██░░░░░░░░ 18ms │ +│ │ └── tx.receive [Node-C] ██████░░░░ 65ms │ +│ └── consensus.round [RCLConsensus] ████████░░ 720ms │ +│ ├── consensus.phase.open ██░░░░░░░░ 180ms │ +│ ├── consensus.phase.establish █████░░░░░ 480ms │ +│ └── consensus.phase.accept █░░░░░░░░░ 60ms │ +└────────────────────────────────────────────────────────────────────────────────┘ +``` + +**RPC Performance Dashboard Panel:** + +``` +┌─────────────────────────────────────────────────────────────┐ +│ RPC Command Latency (Last 1 Hour) │ +├─────────────────────────────────────────────────────────────┤ +│ Command │ p50 │ p95 │ p99 │ Errors │ Rate │ +│──────────────────┼────────┼────────┼────────┼────────┼──────│ +│ account_info │ 12ms │ 45ms │ 89ms │ 0.1% │ 150/s│ +│ submit │ 35ms │ 120ms │ 250ms │ 2.3% │ 45/s│ +│ ledger │ 8ms │ 25ms │ 55ms │ 0.0% │ 80/s│ +│ tx │ 15ms │ 50ms │ 100ms │ 0.5% │ 60/s│ +│ server_info │ 5ms │ 12ms │ 20ms │ 0.0% │ 200/s│ +└─────────────────────────────────────────────────────────────┘ +``` + +**Consensus Health Dashboard Panel:** + +```mermaid +--- +config: + xyChart: + width: 1200 + height: 400 + plotReservedSpacePercent: 50 + chartOrientation: vertical + themeVariables: + xyChart: + plotColorPalette: "#3498db" +--- +xychart-beta + title "Consensus Round Duration (Last 24 Hours)" + x-axis "Time of Day (Hours)" [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24] + y-axis "Duration (seconds)" 1 --> 5 + line [2.1, 2.4, 2.8, 3.2, 3.8, 4.3, 4.5, 5.0, 4.7, 4.0, 3.2, 2.6, 2.0] +``` + +### 1.8.4 Operator Actionable Insights + +| Scenario | What You'll See | Action | +| ------------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------ | +| **Slow RPC** | Span showing which phase is slow (parsing, execution, serialization) | Optimize specific code path | +| **Transaction Stuck** | Trace stops at validation; error attribute shows reason | Fix transaction parameters | +| **Consensus Delay** | Phase.establish taking too long; proposer attribute shows missing validators | Investigate network connectivity | +| **Memory Spike** | Large batch of spans correlating with memory increase | Tune batch_size or sampling | +| **Network Partition** | Traces missing cross-node links for specific peer | Check peer connectivity | +| **Path Computation Slow** | pathfind.compute span shows high latency; cache miss rate in attributes | Warm the RippleLineCache, check order book depth | +| **TxQ Full** | txq.enqueue spans show evictions; fee.escalate spans increasing | Monitor fee levels, alert operators | +| **Ledger Sync Stalled** | ledger.acquire spans timing out; peer reliability attributes show issues | Check peer connectivity, add trusted peers | +| **UNL Stale** | validator.list.fetch spans failing; last_update attribute aging | Verify validator site URLs, check DNS | + +### 1.8.5 Developer Debugging Workflow + +1. **Find Transaction**: Query by `tx_hash` to get full trace +2. **Identify Bottleneck**: Look at span durations to find slowest component +3. **Check Attributes**: Review `validity`, `rpc_status` for errors +4. **Correlate Logs**: Use `trace_id` to find related PerfLog entries +5. **Compare Nodes**: Filter by `service.instance.id` to compare behavior across nodes + +--- + +_Next: [Design Decisions](./02-design-decisions.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/02-design-decisions.md b/OpenTelemetryPlan/02-design-decisions.md new file mode 100644 index 00000000000..9bfee122c68 --- /dev/null +++ b/OpenTelemetryPlan/02-design-decisions.md @@ -0,0 +1,686 @@ +# Design Decisions + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Architecture Analysis](./01-architecture-analysis.md) + +--- + +## 2.1 OpenTelemetry Components + +> **OTLP** = OpenTelemetry Protocol + +### 2.1.1 SDK Selection + +**Primary Choice**: OpenTelemetry C++ SDK (`opentelemetry-cpp`) + +| Component | Purpose | Required | +| --------------------------------------- | ---------------------- | ------------------------- | +| `opentelemetry-cpp::api` | Tracing API headers | Yes | +| `opentelemetry-cpp::sdk` | SDK implementation | Yes | +| `opentelemetry-cpp::ext` | Extensions (exporters) | Yes | +| `opentelemetry-cpp::otlp_http_exporter` | OTLP/HTTP export | Yes (shipped in Phase 1b) | +| `opentelemetry-cpp::otlp_grpc_exporter` | OTLP/gRPC export | Future (not yet wired up) | + +### 2.1.2 Instrumentation Strategy + +**Manual Instrumentation** (recommended): + +| Approach | Pros | Cons | +| ---------- | --------------------------------------------------------------- | ------------------------------------------------------- | +| **Manual** | Precise control, optimized placement, xrpld-specific attributes | More development effort | +| **Auto** | Less code, automatic coverage | Less control, potential overhead, limited customization | + +--- + +## 2.2 Exporter Configuration + +> **OTLP** = OpenTelemetry Protocol + +```mermaid +flowchart TB + subgraph nodes["xrpld Nodes"] + node1["xrpld
Node 1"] + node2["xrpld
Node 2"] + node3["xrpld
Node 3"] + end + + collector["OpenTelemetry
Collector
(sidecar or standalone)"] + + subgraph backends["Observability Backends"] + tempo["Tempo"] + elastic["Elastic
APM"] + end + + node1 -->|"OTLP/HTTP
:4318"| collector + node2 -->|"OTLP/HTTP
:4318"| collector + node3 -->|"OTLP/HTTP
:4318"| collector + + collector --> tempo + collector --> elastic + + style nodes fill:#0d47a1,stroke:#082f6a,color:#ffffff + style backends fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style collector fill:#bf360c,stroke:#8c2809,color:#ffffff +``` + +**Reading the diagram:** + +- **xrpld Nodes (blue)**: The source of telemetry data. Each xrpld node exports spans via OTLP/HTTP on port 4318 (the only exporter shipped in Phase 1b). +- **OpenTelemetry Collector (red)**: The central aggregation point that receives spans from all nodes. Can run as a sidecar (per-node) or standalone (shared). Handles batching, filtering, and routing. +- **Observability Backends (green)**: The storage and visualization destinations. Tempo is the recommended backend for both development and production, and Elastic APM is an alternative. The Collector routes to one or more backends. +- **Arrows (nodes to collector to backends)**: The data pipeline -- spans flow from nodes to the Collector over HTTP, then the Collector fans out to the configured backends. + +### 2.2.1 OTLP/HTTP (Shipped in Phase 1b) + +OTLP/HTTP is the only exporter wired up in Phase 1b. It is configured via +`OtlpHttpExporterOptions` with the collector traces endpoint +(`http://localhost:4318/v1/traces` by default) and a JSON content type +(binary protobuf is also available). + +### 2.2.2 OTLP/gRPC (Future Work — Planned Upgrade) + +OTLP/gRPC is planned as a future upgrade from the HTTP exporter. The gRPC +transport offers lower per-span overhead and tighter back-pressure semantics +than HTTP/JSON, making it attractive for production deployments once the HTTP +path is validated in earlier phases. + +Required to land this upgrade: + +1. Add `opentelemetry-cpp::otlp_grpc_exporter` to the Conan recipe (the + dependency already exists but is not linked in Phase 1b builds). +2. Extend `TelemetryConfig.cpp` to parse an `exporter` key (`otlp_http` + default, `otlp_grpc` opt-in) and a gRPC endpoint override. +3. In `Telemetry::start()` branch on the parsed exporter type and construct + either `OtlpHttpExporterFactory::Create(httpOpts)` or + `OtlpGrpcExporterFactory::Create(grpcOpts)` accordingly. +4. Update the runbook and dashboards to document the alternate port and TLS + settings. + +When wired up, the gRPC path will use `OtlpGrpcExporterOptions` configured with +the collector endpoint (host on port 4317), TLS credentials enabled, and a CA +certificate path. + +Until that work lands, `OtlpGrpcExporterOptions` is **not** used by any code +path in Phase 1b through Phase 5. + +--- + +## 2.3 Span Naming Conventions + +> **TxQ** = Transaction Queue | **UNL** = Unique Node List | **WS** = WebSocket + +### 2.3.1 Naming Schema + +``` +.[.] +``` + +**Examples**: + +- `tx.receive` - Transaction received from peer +- `consensus.phase.establish` - Consensus establish phase +- `rpc.command.server_info` - server_info RPC command + +### 2.3.2 Complete Span Catalog + +| Span name | Description | +| ------------------------------ | --------------------------------------- | +| `tx.receive` | Transaction received from network | +| `tx.validate` | Transaction signature/format validation | +| `tx.process` | Full transaction processing | +| `tx.relay` | Transaction relay to peers | +| `tx.apply` | Apply transaction to ledger | +| `consensus.round` | Complete consensus round | +| `consensus.phase.open` | Open phase - collecting transactions | +| `consensus.phase.establish` | Establish phase - reaching agreement | +| `consensus.phase.accept` | Accept phase - applying consensus | +| `consensus.proposal.receive` | Receive peer proposal | +| `consensus.proposal.send` | Send our proposal | +| `consensus.validation.receive` | Receive peer validation | +| `consensus.validation.send` | Send our validation | +| `rpc.request` | HTTP/WebSocket request handling | +| `rpc.command.*` | Specific RPC command (dynamic) | +| `peer.connect` | Peer connection establishment | +| `peer.disconnect` | Peer disconnection | +| `peer.message.send` | Send protocol message | +| `peer.message.receive` | Receive protocol message | +| `ledger.acquire` | Ledger acquisition from network | +| `ledger.build` | Build new ledger | +| `ledger.validate` | Ledger validation | +| `ledger.close` | Close ledger | +| `ledger.replay` | Ledger replay executed | +| `ledger.delta` | Delta-based ledger acquired | +| `pathfind.request` | Path request initiated | +| `pathfind.compute` | Path computation executed | +| `txq.enqueue` | Transaction queued | +| `txq.apply` | Queued transaction applied | +| `fee.escalate` | Fee escalation triggered | +| `validator.list.fetch` | UNL list fetched | +| `validator.manifest` | Manifest update processed | +| `amendment.vote` | Amendment voting executed | +| `shamap.sync` | State tree synchronization | +| `job.enqueue` | Job added to queue | +| `job.execute` | Job execution | + +### 2.3.3 Attribute Naming Conventions + +Span **names** follow §2.3.1 (dotted `.`). Span +**attribute keys** follow the rules below. The constants in the `*SpanNames.h` +headers are the single source of truth; the collector, Tempo, the Grafana +dashboards, and the runbook all consume these exact keys, so every layer must +agree with the code. A CI check enforces this end to end. + +1. **Per-span unique attribute** → bare field name, allowed when the field is + recorded by a single span/workflow so the span name already supplies the + domain (e.g. `command`, `version`, `local` on `rpc.command`). +2. **Shared attribute (same concept on more than one span)** → ONE key, reused + verbatim on every span that records it; the span name tells the occurrences + apart, so no per-emitter prefix is added. Name it by the field's meaning: a + property of a domain object keeps that object's bare field name (`ledger_hash`, + `ledger_seq`, `tx_hash`, `peer_id`, `full_validation`); a field already + qualified by a sub-kind keeps that qualifier on every emitter (`proposal_trusted` + on both `consensus.proposal.receive` and `peer.proposal.receive`; + `validation_trusted` likewise). Defined once in the base `SpanNames.h` + `namespace attr` block and re-exported (`using`) by each domain header. +3. **Collision qualifier** → `_`, only when a bare name would + collide with a DIFFERENT concept in the shared spanmetrics label space or with + the OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`, + `consensus_phase`, `consensus_round`, `consensus_mode`). This disambiguates + distinct concepts that share a word; it is NOT used to tag the same concept + with its emitting workflow — that is rule 2 (one shared name). +4. **Resource attribute** → dotted `xrpl..`, reserved ONLY + for process/network identity set once at startup (`xrpl.network.id`, + `xrpl.network.type`). Span attributes are never dotted in the `xrpl.` form — + it blurs the resource/span scope boundary and parses awkwardly in TraceQL. +5. **Span names** use `[.]` (dotted, per §2.3.1). Only + attribute _keys_ follow rules 1–4. + +Standard OpenTelemetry semantic-convention keys keep their canonical dotted +form (e.g. `service.*` resource attributes, `http.*` span attributes); the +"no dotted form" rule applies to xrpl-custom keys only. + +The same rules are recorded in `CONTRIBUTING.md` (the permanent home, since +`OpenTelemetryPlan/` is removed once the rollout completes). The attribute +examples in §2.4 below follow these rules. + +--- + +## 2.4 Attribute Schema + +> **TxQ** = Transaction Queue | **UNL** = Unique Node List | **OTLP** = OpenTelemetry Protocol + +### 2.4.1 Resource Attributes (Set Once at Startup) + +Resource attributes identify the process and are set once at startup. They use +the standard OpenTelemetry semantic conventions plus custom dotted `xrpl.*` +keys (the dotted form is reserved for resource scope per §2.3.3). + +| Key | Type / value | Description | +| --------------------- | ------------------------------------------------------- | ------------------------------ | +| `service.name` | `"xrpld"` | Standard `SERVICE_NAME` | +| `service.version` | `BuildInfo::getVersionString()` | Standard `SERVICE_VERSION` | +| `service.instance.id` | node public key (base58) | Standard `SERVICE_INSTANCE_ID` | +| `xrpl.network.id` | network id (e.g. 0 for mainnet) | Network identifier | +| `xrpl.network.type` | `"mainnet"` \| `"testnet"` \| `"devnet"` \| `"unknown"` | Network kind | +| `xrpl.node.type` | `"validator"` \| `"stock"` \| `"reporting"` | Node role | +| `xrpl.node.cluster` | cluster name | Cluster name, if clustered | + +### 2.4.2 Span Attributes by Category + +> Span attribute keys use the underscore form from §2.3.3 (shared/qualified +> keys are `_`; per-span unique keys are bare). The dotted form +> is reserved for the resource attributes in §2.4.1 above. This catalog lists +> the planned attribute set by category; the exact emitted key for each +> implemented span is defined by the `*SpanNames.h` constants, which are the +> single source of truth where the two differ. + +#### Transaction Attributes + +| Key | Type | Description | +| -------------- | ------ | ------------------------------------- | +| `tx_hash` | string | Transaction hash (hex) | +| `tx_type` | string | `"Payment"`, `"OfferCreate"`, etc. | +| `tx_account` | string | Source account (redacted in prod) | +| `tx_sequence` | int64 | Account sequence number | +| `tx_fee` | int64 | Fee in drops | +| `tx_result` | string | `"tesSUCCESS"`, `"tecPATH_DRY"`, etc. | +| `ledger_index` | int64 | Ledger containing transaction | +| `relay_count` | int64 | Peers the transaction was relayed to | +| `suppressed` | bool | `true` when HashRouter dropped a dup | + +#### Consensus Attributes + +| Key | Type | Description | +| -------------------- | ------- | ----------------------------------- | +| `consensus_round` | int64 | Round number | +| `consensus_phase` | string | `"open"`, `"establish"`, `"accept"` | +| `consensus_mode` | string | `"proposing"`, `"observing"`, etc. | +| `proposers` | int64 | Number of proposers | +| `prev_ledger_prefix` | string | Previous ledger hash prefix | +| `ledger_seq` | int64 | Ledger sequence | +| `tx_count` | int64 | Transactions in consensus set | +| `round_time_ms` | float64 | Round duration | + +Establish-phase gap fill and cross-node correlation attributes (Phase 4a): + +| Key | Type | Description | +| --------------------- | ------ | --------------------------------------------------------- | +| `consensus_round_id` | int64 | Consensus round number | +| `consensus_ledger_id` | string | `previousLedger.id()` — shared across nodes | +| `trace_strategy` | string | `"deterministic"` or `"attribute"` | +| `converge_percent` | int64 | Convergence % (0-100+) | +| `establish_count` | int64 | Number of establish iterations | +| `disputes_count` | int64 | Active disputed transactions | +| `agree_count` | int64 | Peers that agree (haveConsensus) | +| `disagree_count` | int64 | Peers that disagree | +| `threshold_percent` | int64 | Close-time consensus threshold (`avCT_CONSENSUS_PCT`=75%) | +| `consensus_result` | string | `"yes"`, `"no"`, `"moved_on"`, `"expired"` | +| `mode_old` | string | Previous consensus mode | +| `mode_new` | string | New consensus mode | + +#### RPC Attributes + +| Key | Type | Description | +| ------------- | ------- | ----------------------------------------------------------------------------- | +| `command` | string | Command name (per-span unique on `rpc.command`) | +| `version` | int64 | API version | +| `rpc_role` | string | `"admin"` or `"user"` (qualified — `role` is generic) | +| `params` | string | Sanitized parameters (optional) | +| `rpc_status` | string | Response status: `success` \| `error` (qualified — `status` is OTel-reserved) | +| `duration_ms` | float64 | Request duration in milliseconds | + +#### Peer & Message Attributes + +| Key | Type | Description | +| -------------------- | ------- | -------------------------- | +| `peer_id` | string | Peer public key (base58) | +| `peer_address` | string | IP:port | +| `peer_latency_ms` | float64 | Measured latency | +| `peer_cluster` | string | Cluster name if clustered | +| `message_type` | string | Protocol message type name | +| `message_size_bytes` | int64 | Message size | +| `message_compressed` | bool | Whether compressed | + +#### Ledger & Job Attributes + +| Key | Type | Description | +| ----------------- | ------- | --------------------- | +| `ledger_hash` | string | Ledger hash | +| `ledger_index` | int64 | Ledger sequence/index | +| `close_time` | int64 | Close time (epoch) | +| `ledger_tx_count` | int64 | Transaction count | +| `job_type` | string | Job type name | +| `job_queue_ms` | float64 | Time spent in queue | +| `job_worker` | int64 | Worker thread ID | + +#### PathFinding Attributes + +| Key | Type | Description | +| -------------------------- | ------ | ------------------------- | +| `pathfind_source_currency` | string | Source currency code | +| `pathfind_dest_currency` | string | Destination currency code | +| `pathfind_path_count` | int64 | Number of paths found | +| `pathfind_cache_hit` | bool | RippleLineCache hit | + +#### TxQ Attributes + +| Key | Type | Description | +| --------------------- | ------ | --------------------------- | +| `txq_queue_depth` | int64 | Current queue depth | +| `txq_fee_level` | int64 | Fee level of transaction | +| `txq_eviction_reason` | string | Why transaction was evicted | + +#### Fee Attributes + +| Key | Type | Description | +| ---------------------- | ----- | ------------------------- | +| `fee_load_factor` | int64 | Current load factor | +| `fee_escalation_level` | int64 | Fee escalation multiplier | + +#### Validator Attributes + +| Key | Type | Description | +| ------------------------ | ----- | ------------------------- | +| `validator_list_size` | int64 | UNL size | +| `validator_list_age_sec` | int64 | Seconds since last update | + +#### Amendment Attributes + +| Key | Type | Description | +| ------------------ | ------ | -------------------------------------- | +| `amendment_name` | string | Amendment name | +| `amendment_status` | string | `"enabled"`, `"vetoed"`, `"supported"` | + +#### SHAMap Attributes + +| Key | Type | Description | +| ---------------------- | ------- | --------------------------------------------- | +| `shamap_type` | string | `"transaction"`, `"state"`, `"account_state"` | +| `shamap_missing_nodes` | int64 | Number of missing nodes during sync | +| `shamap_duration_ms` | float64 | Sync duration | + +### 2.4.3 Data Collection Summary + +The following table summarizes what data is collected by category: + +| Category | Attributes Collected | Purpose | +| --------------- | ---------------------------------------------------------------------------------------------------------------- | ---------------------------- | +| **Transaction** | `tx_hash`, `tx_type`, `tx_result`, `tx_fee`, `ledger_index` | Trace transaction lifecycle | +| **Consensus** | `consensus_round`, `consensus_phase`, `consensus_mode`, `proposers`, `round_time_ms` | Analyze consensus timing | +| **RPC** | `command`, `version`, `rpc_status`, `duration_ms` | Monitor RPC performance | +| **Peer** | `peer_id` (public key), `peer_latency_ms`, `message_type`, `message_size_bytes` | Network topology analysis | +| **Ledger** | `ledger_hash`, `ledger_index`, `close_time`, `ledger_tx_count` | Ledger progression tracking | +| **Job** | `job_type`, `job_queue_ms`, `job_worker` | JobQueue performance | +| **PathFinding** | `pathfind_fast`, `pathfind_search_level`, `pathfind_num_paths`, `pathfind_ledger_index`, `pathfind_num_requests` | Payment path analysis | +| **TxQ** | `txq_queue_depth`, `txq_fee_level`, `txq_eviction_reason` | Queue depth and fee tracking | +| **Fee** | `fee_load_factor`, `fee_escalation_level` | Fee escalation monitoring | +| **Validator** | `validator_list_size`, `validator_list_age_sec` | UNL health monitoring | +| **Amendment** | `amendment_name`, `amendment_status` | Protocol upgrade tracking | +| **SHAMap** | `shamap_type`, `shamap_missing_nodes`, `shamap_duration_ms` | State tree sync performance | + +### 2.4.4 Privacy & Sensitive Data Policy + +> **PII** = Personally Identifiable Information + +OpenTelemetry instrumentation is designed to collect **operational metadata only**, never sensitive content. + +#### Data NOT Collected + +The following data is explicitly **excluded** from telemetry collection: + +| Excluded Data | Reason | +| ----------------------- | ----------------------------------------- | +| **Private Keys** | Never exposed; not relevant to tracing | +| **Account Balances** | Financial data; privacy sensitive | +| **Transaction Amounts** | Financial data; privacy sensitive | +| **Raw TX Payloads** | May contain sensitive memo/data fields | +| **Personal Data** | No PII collected | +| **IP Addresses** | Configurable; excluded by default in prod | + +#### Privacy Protection Mechanisms + +| Mechanism | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Account Hashing** | Account addresses are hashed both SDK-side (`pathfind_source_account`, `pathfind_dest_account` — always hashed before emission) and again at the collector level, so raw addresses never reach storage | +| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section | +| **Collector Tail Sampling** | xrpld head sampling is fixed at 1.0 (every span emitted); the collector retains ~10% of non-error traces, reducing stored data exposure | +| **Sampling** | Only 10% of traces recorded by default, reducing data exposure | +| **Local Control** | Node operators have full control over what gets exported | +| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | +| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | + +#### Account Address Hashing + +Account addresses are **always** hashed before they reach the telemetry +backend — there is no opt-out flag and therefore no insecure-by-default +failure mode. Protection is applied in two independent layers: + +1. **SDK-side** (this node): the path-finding RPC handlers call + `redactAccount()` (`xrpl::telemetry`, `Redaction.h`) before setting the + `pathfind_source_account` / `pathfind_dest_account` span attributes. The + helper emits the first 16 characters of `sha512Half(address)` as + lowercase hex — deterministic (spans for one account still correlate) + but non-reversible. +2. **Collector-side** (defense-in-depth): an `attributes/hash` processor in + the OpenTelemetry Collector re-hashes those same attributes, so any node + that emitted a raw value is still redacted before storage. + +#### Collector-Level Data Protection + +The OpenTelemetry Collector can be configured (via an `attributes` processor) +to hash or redact sensitive attributes before export — for example, hashing +`pathfind_source_account` / `pathfind_dest_account`, deleting `peer_address` +to drop IP addresses, and deleting `params` to redact request parameters. + +#### Configuration Options for Privacy + +In `xrpld.cfg`, operators control data collection granularity through the +`[telemetry]` section. Besides `enabled`, per-component toggles +(`trace_transactions`, `trace_consensus`, `trace_rpc`, `trace_peer` — the last +often disabled due to high volume) select which spans are emitted. Account +address hashing is not configurable: addresses are hashed unconditionally by +the SDK helper described above, with collector-level hashing as a second +layer. + +> **Key Principle**: Telemetry collects **operational metadata** (timing, counts, hashes) — never **sensitive content** (keys, balances, amounts, raw payloads). + +> **See also**: [Securing the OTel Pipeline](./secure-OTel.md) covers transport-level protection for telemetry leaving the node — mTLS to the collector and validation of incoming peer trace context. Privacy controls in this section keep sensitive data out of spans; the security doc keeps the spans themselves out of untrusted hands. + +--- + +## 2.5 Context Propagation Design + +> **WS** = WebSocket + +### 2.5.0 Deterministic Trace ID Strategy + +Both transaction and consensus tracing use **deterministic trace IDs** derived from +a globally known hash, so all nodes handling the same workflow independently produce +spans under the same `trace_id`. This is combined with protobuf `span_id` propagation +for parent-child relay ordering when available. + +#### Transactions — `trace_id = txHash[0:16]` + +Every node that handles a transaction knows its `txID` (the `uint256` transaction +hash). The first 16 bytes of this hash are used as the OTel `trace_id`: + +``` +uint256 txHash: A1B2C3D4 E5F6A7B8 C9D0E1F2 A3B4C5D6 E7F8A9B0 C1D2E3F4 A5B6C7D8 E9F0A1B2 + |---------- trace_id (16 bytes) ---------| (remaining 16 bytes unused) +``` + +Each node generates a **random 8-byte `span_id`** so its span is unique within the +shared trace. When protobuf `TraceContext` is present in the incoming `TMTransaction`, +the sender's `span_id` is extracted and used as the parent — preserving the relay +chain as a parent-child tree. When absent (older peers, first hop from client), the +span appears as a root in the same trace — correlation is preserved, only the tree +structure degrades. + +``` +Node A (submitter) Node B (relay) Node C (relay) +trace_id: A1B2... trace_id: A1B2... trace_id: A1B2... +span_id: 1234 (random) span_id: 5678 (random) span_id: 9ABC (random) +parent: (none) parent: 1234 (proto) parent: 5678 (proto) + ↑ ↑ + protobuf propagation protobuf propagation +``` + +If protobuf propagation fails at Node B (old peer): + +``` +Node A Node B (old peer) Node C +trace_id: A1B2... trace_id: A1B2... trace_id: A1B2... +span_id: 1234 span_id: 5678 span_id: 9ABC +parent: (none) parent: (none) parent: 5678 (proto) + ↑ no parent, but same trace_id — still grouped +``` + +#### Consensus — `trace_id = prevLedgerHash[0:16]` + +All validators in the same consensus round share the same `previousLedger.id()`. +The first 16 bytes are used as trace_id. See [Phase 4a implementation status](./06-implementation-phases.md) +and `createDeterministicContext()` in `RCLConsensus.cpp` for the implementation. + +Switchable via `consensus_trace_strategy` config: +`"deterministic"` (default) or `"attribute"` (random trace_id, correlation via attribute queries). + +#### Why Not Random IDs with Propagation Only? + +Random trace IDs require **unbroken context propagation** across every hop. In a +mixed-version network (common during upgrades), older peers silently drop the +`trace_context` protobuf field. The trace splits and downstream spans become +impossible to find. Deterministic IDs make correlation **propagation-resilient** — the trace +backend groups all spans for the same transaction/round regardless of whether +propagation succeeded. + +#### Why Keep Protobuf Propagation? + +Deterministic trace IDs alone provide correlation (all spans grouped) but not +**causality** (which node relayed to which). Protobuf `span_id` propagation adds +parent-child ordering that shows the exact relay path. The two mechanisms complement +each other: + +| Mechanism | Provides | Fails when | +| ---------------------------- | --------------------------- | -------------------------------------- | +| Deterministic trace_id | Cross-node correlation | Never (hash is always known) | +| Protobuf span_id propagation | Parent-child relay ordering | Older peer drops `trace_context` field | + +#### Implementation Reference + +The utility function `createDeterministicTxContext(uint256 const& txHash)` follows +the same pattern as `createDeterministicContext(uint256 const& ledgerId)` in +`RCLConsensus.cpp`. See [Phase 3 Task 3.9](./Phase3_taskList.md) for the full spec. + +### 2.5.1 Propagation Boundaries + +```mermaid +flowchart TB + subgraph http["HTTP/WebSocket (RPC)"] + w3c["W3C Trace Context Headers:
traceparent:
00-trace_id-span_id-flags
tracestate: xrpld=..."] + end + + subgraph protobuf["Protocol Buffers (P2P)"] + proto["message TraceContext {
bytes trace_id = 1; // 16 bytes
bytes span_id = 2; // 8 bytes
uint32 trace_flags = 3;
string trace_state = 4;
}"] + end + + subgraph jobqueue["JobQueue (Internal Async)"] + job["Context captured at job creation,
restored at execution

class Job {
otel::context::Context
traceContext_;
};"] + end + + style http fill:#0d47a1,stroke:#082f6a,color:#ffffff + style protobuf fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style jobqueue fill:#bf360c,stroke:#8c2809,color:#ffffff +``` + +**Reading the diagram:** + +- **HTTP/WebSocket - RPC (blue)**: For client-facing RPC requests, trace context is propagated using the W3C `traceparent` header. This is the standard approach and works with any OTel-compatible client. +- **Protocol Buffers - P2P (green)**: For peer-to-peer messages between xrpld nodes, trace context is embedded as a protobuf `TraceContext` message carrying trace_id, span_id, flags, and optional trace_state. +- **JobQueue - Internal Async (red)**: For asynchronous work within a single node, the OTel context is captured when a job is created and restored when the job executes on a worker thread. This bridges the async gap so spans remain linked. + +--- + +## 2.6 Integration with Existing Observability + +> **OTLP** = OpenTelemetry Protocol | **WS** = WebSocket + +### 2.6.1 Existing Frameworks Comparison + +xrpld already has two observability mechanisms. OpenTelemetry complements (not replaces) them: + +| Aspect | PerfLog | Beast Insight (StatsD) | OpenTelemetry | +| --------------------- | ----------------------------- | ---------------------------- | ------------------------- | +| **Type** | Logging | Metrics | Distributed Tracing | +| **Data** | JSON log entries | Counters, gauges, histograms | Spans with context | +| **Scope** | Single node | Single node | **Cross-node** | +| **Output** | `perf.log` file | StatsD server | OTLP Collector | +| **Question answered** | "What happened on this node?" | "How many? How fast?" | "What was the journey?" | +| **Correlation** | By timestamp | By metric name | By `trace_id` | +| **Overhead** | Low (file I/O) | Low (UDP packets) | Low-Medium (configurable) | + +### 2.6.2 What Each Framework Does Best + +#### PerfLog + +- **Purpose**: Detailed local event logging for RPC and job execution +- **Strengths**: + - Rich JSON output with timing data + - Already integrated in RPC handlers + - File-based, no external dependencies +- **Limitations**: + - Single-node only (no cross-node correlation) + - No parent-child relationships between events + - Manual log parsing required + +A PerfLog entry is a JSON object with fields such as `time`, `method`, +`duration_us`, and `result`. + +#### Beast Insight (StatsD) + +- **Purpose**: Real-time metrics for monitoring dashboards +- **Strengths**: + - Aggregated metrics (counters, gauges, histograms) + - Low overhead (UDP, fire-and-forget) + - Good for alerting thresholds +- **Limitations**: + - No request-level detail + - No causal relationships + - Single-node perspective + +In xrpld, Beast Insight is used through `increment` (counters), `gauge` +(point-in-time values), and `timing` (durations) calls. + +#### OpenTelemetry (NEW) + +- **Purpose**: Distributed request tracing across nodes +- **Strengths**: + - **Cross-node correlation** via `trace_id` + - Parent-child span relationships + - Rich attributes per span + - Industry standard (CNCF) +- **Limitations**: + - Requires collector infrastructure + - Higher complexity than logging + +A span is created via `startSpan` (e.g. `"tx.relay"`), annotated with +attributes such as `tx_hash` and `peer_id`, and is automatically linked to its +parent through the active context. + +### 2.6.3 When to Use Each + +| Scenario | PerfLog | StatsD | OpenTelemetry | +| --------------------------------------- | ---------- | ------ | ------------- | +| "How many TXs per second?" | ❌ | ✅ | ✅ | +| "What's the p99 RPC latency?" | ❌ | ✅ | ✅ | +| "Why was this specific TX slow?" | ⚠️ partial | ❌ | ✅ | +| "Which node delayed consensus?" | ❌ | ❌ | ✅ | +| "What happened on node X at time T?" | ✅ | ❌ | ✅ | +| "Show me the TX journey across 5 nodes" | ❌ | ❌ | ✅ | + +### 2.6.4 Coexistence Strategy + +> **Note**: Phase 7 replaces the StatsD bridge with native OTel Metrics SDK export. The diagram below shows the Phase 6 intermediate state. See [Phase7_taskList.md](./Phase7_taskList.md) for the migration design where Beast Insight emits via OTLP instead of StatsD. + +```mermaid +flowchart TB + subgraph xrpld["xrpld Process"] + perflog["PerfLog
(JSON to file)"] + insight["Beast Insight
(StatsD)"] + otel["OpenTelemetry
(Tracing)"] + end + + perflog --> perffile["perf.log"] + insight --> statsd["StatsD Server"] + otel --> collector["OTLP Collector"] + + perffile --> grafana["Grafana
(Unified UI)"] + statsd --> grafana + collector --> grafana + + style xrpld fill:#212121,stroke:#0a0a0a,color:#ffffff + style grafana fill:#bf360c,stroke:#8c2809,color:#ffffff +``` + +**Reading the diagram:** + +- **xrpld Process (dark gray)**: The single xrpld node running all three observability frameworks side by side. Each framework operates independently with no interference. +- **PerfLog to perf.log**: PerfLog writes JSON-formatted event logs to a local file. Grafana can ingest these via Loki or a file-based datasource. +- **Beast Insight to StatsD Server**: Insight sends aggregated metrics (counters, gauges) over UDP to a StatsD server. Grafana reads from StatsD-compatible backends like Graphite or Prometheus (via StatsD exporter). +- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/HTTP to a Collector, which then forwards to a trace backend (Tempo). (OTLP/gRPC is future work — §2.2.2.) +- **Grafana (red, unified UI)**: All three data streams converge in Grafana, enabling operators to correlate logs, metrics, and traces in a single dashboard. + +**Phase 7 target state**: Beast Insight routes to `OTelCollector` (new `Collector` implementation) which exports via OTLP/HTTP to the same collector endpoint as traces. StatsD UDP path becomes a deprecated fallback (`[insight] server=statsd`). See [06-implementation-phases.md §6.8](./06-implementation-phases.md) and [Phase7_taskList.md](./Phase7_taskList.md) for details. + +### 2.6.5 Correlation with PerfLog + +Trace IDs can be correlated with existing PerfLog entries for comprehensive +debugging. The design is for `RPCHandler.cpp` to start an `rpc.command.` +span alongside the existing PerfLog `rpcStart`/`rpcFinish`/`rpcError` calls, +extract the span's `trace_id` (when valid), and eventually stamp it onto the +PerfLog entry (a planned `setTraceId` hook) so logs and traces share a key. The +span status is set to OK on success or to error (recording the exception) on +failure. + +--- + +_Previous: [Architecture Analysis](./01-architecture-analysis.md)_ | _Next: [Implementation Strategy](./03-implementation-strategy.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/03-implementation-strategy.md b/OpenTelemetryPlan/03-implementation-strategy.md new file mode 100644 index 00000000000..922d3d4999a --- /dev/null +++ b/OpenTelemetryPlan/03-implementation-strategy.md @@ -0,0 +1,483 @@ +# Implementation Strategy + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Configuration Reference](./05-configuration-reference.md) + +--- + +## 3.1 Directory Structure + +The telemetry implementation follows xrpld's existing code organization pattern: + +``` +include/xrpl/ +├── telemetry/ +│ ├── Telemetry.h # Main telemetry interface (global singleton) +│ ├── TelemetryConfig.h # Configuration structures +│ ├── TraceContext.h # Context propagation utilities +│ ├── SpanGuard.h # RAII span management with factory methods + discard() +│ ├── DiscardFlag.h # Thread-local discard flag +│ └── SpanAttributes.h # Attribute helper functions + +src/libxrpl/ +├── telemetry/ +│ ├── Telemetry.cpp # Implementation + FilteringSpanProcessor +│ ├── TelemetryConfig.cpp # Config parsing +│ ├── TraceContext.cpp # Context serialization +│ └── NullTelemetry.cpp # No-op implementation +``` + +--- + +## 3.2 Implementation Approach + +
+ +```mermaid +%%{init: {'flowchart': {'nodeSpacing': 20, 'rankSpacing': 30}}}%% +flowchart TB + subgraph phase1["Phase 1: Core"] + direction LR + sdk["SDK Integration"] ~~~ interface["Telemetry Interface"] ~~~ config["Configuration"] + end + + subgraph phase2["Phase 2: RPC"] + direction LR + http["HTTP Context"] ~~~ rpc["RPC Handlers"] + end + + subgraph phase3["Phase 3: P2P"] + direction LR + proto["Protobuf Context"] ~~~ tx["Transaction Relay"] + end + + subgraph phase4["Phase 4: Consensus"] + direction LR + consensus["Consensus Rounds"] ~~~ proposals["Proposals"] + end + + phase1 --> phase2 --> phase3 --> phase4 + + style phase1 fill:#1565c0,stroke:#0d47a1,color:#ffffff + style phase2 fill:#2e7d32,stroke:#1b5e20,color:#ffffff + style phase3 fill:#e65100,stroke:#bf360c,color:#ffffff + style phase4 fill:#c2185b,stroke:#880e4f,color:#ffffff +``` + +
+ +### Key Principles + +1. **Minimal Intrusion**: Instrumentation should not alter existing control flow +2. **Zero-Cost When Disabled**: Use compile-time flags and no-op implementations +3. **Backward Compatibility**: Protocol Buffer extensions use high field numbers +4. **Graceful Degradation**: Tracing failures must not affect node operation + +--- + +## 3.3 Performance Overhead Summary + +> **OTLP** = OpenTelemetry Protocol + +| Metric | Overhead | Notes | +| ------------- | ---------- | ------------------------------------------------ | +| CPU | 1-3% | Of per-transaction CPU cost (~200μs baseline) | +| Memory | ~10 MB | SDK statics + batch buffer + worker thread stack | +| Network | 10-50 KB/s | Compressed OTLP export to collector | +| Latency (p99) | <2% | With proper sampling configuration | + +--- + +## 3.4 Detailed CPU Overhead Analysis + +### 3.4.1 Per-Operation Costs + +> **Note on hardware assumptions**: The costs below are based on the official OTel C++ SDK CI benchmarks +> (969 runs on GitHub Actions 2-core shared runners). On production server hardware (3+ GHz Xeon), +> expect costs at the **lower end** of each range (~30-50% improvement over CI hardware). + +| Operation | Time (ns) | Frequency | Impact | +| --------------------- | --------- | ---------------------- | ---------- | +| Span creation | 500-1000 | Every traced operation | Low | +| Span end | 100-200 | Every traced operation | Low | +| SetAttribute (string) | 80-120 | 3-5 per span | Low | +| SetAttribute (int) | 40-60 | 2-3 per span | Negligible | +| AddEvent | 100-200 | 0-2 per span | Low | +| Context injection | 150-250 | Per outgoing message | Low | +| Context extraction | 100-180 | Per incoming message | Low | +| GetCurrent context | 10-20 | Thread-local access | Negligible | + +**Source**: Span creation based on OTel C++ SDK `BM_SpanCreation` benchmark (AlwaysOnSampler + +SimpleSpanProcessor + InMemoryExporter), median ~1,000 ns on CI hardware. AddEvent includes +timestamp read + string copy + vector push + mutex acquisition. Context injection/extraction +confirmed by `BM_SpanCreationWithScope` benchmark delta (~160 ns). + +### 3.4.2 Transaction Processing Overhead + +
+ +```mermaid +%%{init: {'pie': {'textPosition': 0.75}}}%% +pie showData + "tx.receive (1400ns)" : 1400 + "tx.validate (1200ns)" : 1200 + "tx.relay (1200ns)" : 1200 + "Context inject (200ns)" : 200 +``` + +**Transaction Tracing Overhead (~4.0μs total)** + +
+ +**Overhead percentage**: 4.0 μs / 200 μs (avg tx processing) = **~2.0%** + +> **Breakdown**: Each span (tx.receive, tx.validate, tx.relay) costs ~1,000 ns for creation plus +> ~200-400 ns for 3-5 attribute sets. Context injection is ~200 ns (confirmed by benchmarks). +> On production hardware, expect ~2.6 μs total (~1.3% overhead) due to faster span creation (~500-600 ns). + +### 3.4.3 Consensus Round Overhead + +| Operation | Count | Cost (ns) | Total | +| ---------------------- | ----- | --------- | ---------- | +| consensus.round span | 1 | ~1200 | ~1.2 μs | +| consensus.phase spans | 3 | ~1100 | ~3.3 μs | +| proposal.receive spans | ~20 | ~1100 | ~22 μs | +| proposal.send spans | ~3 | ~1100 | ~3.3 μs | +| Context operations | ~30 | ~200 | ~6 μs | +| **TOTAL** | | | **~36 μs** | + +> **Why higher**: Each span costs ~1,000 ns creation + ~100-200 ns for 1-2 attributes, totaling ~1,100-1,200 ns. +> Context operations remain ~200 ns (confirmed by benchmarks). On production hardware, expect ~24 μs total. + +**Overhead percentage**: 36 μs / 3s (typical round) = **~0.001%** (negligible) + +### 3.4.4 RPC Request Overhead + +| Operation | Cost (ns) | +| ---------------- | ------------ | +| rpc.request span | ~1200 | +| rpc.command span | ~1100 | +| Context extract | ~250 | +| Context inject | ~200 | +| **TOTAL** | **~2.75 μs** | + +> **Why higher**: Each span costs ~1,000 ns creation + ~100-200 ns for attributes (command name, +> version, role). Context extract/inject costs are confirmed by OTel C++ benchmarks. + +- Fast RPC (1ms): 2.75 μs / 1ms = **~0.275%** +- Slow RPC (100ms): 2.75 μs / 100ms = **~0.003%** + +--- + +## 3.5 Memory Overhead Analysis + +> **OTLP** = OpenTelemetry Protocol + +### 3.5.1 Static Memory + +| Component | Size | Allocated | +| ------------------------------------ | ----------- | ---------- | +| TracerProvider singleton | ~64 KB | At startup | +| BatchSpanProcessor (circular buffer) | ~16 KB | At startup | +| BatchSpanProcessor (worker thread) | ~8 MB | At startup | +| OTLP/HTTP exporter (client init) | ~64 KB | At startup | +| Propagator registry | ~8 KB | At startup | +| **Total static** | **~8.1 MB** | | + +> **Why higher than earlier estimate**: The BatchSpanProcessor's circular buffer itself is only ~16 KB +> (2049 x 8-byte `AtomicUniquePtr` entries), but it spawns a dedicated worker thread whose default +> stack size on Linux is ~8 MB. The OTLP/HTTP exporter allocates a small client and TLS +> initialization buffer. The worker thread stack dominates the static footprint. + +### 3.5.2 Dynamic Memory + +| Component | Size per unit | Max units | Peak | +| -------------------- | -------------- | ---------- | --------------- | +| Active span | ~500-800 bytes | 1000 | ~500-800 KB | +| Queued span (export) | ~500 bytes | 2048 | ~1 MB | +| Attribute storage | ~80 bytes | 5 per span | Included | +| Context storage | ~64 bytes | Per thread | ~6.4 KB | +| **Total dynamic** | | | **~1.5-1.8 MB** | + +> **Why active spans are larger**: An active `Span` object includes the wrapper (~88 bytes: shared_ptr, +> mutex, unique_ptr to Recordable) plus `SpanData` (~250 bytes: SpanContext, timestamps, name, status, +> empty containers) plus attribute storage (~200-500 bytes for 3-5 string attributes in a `std::map`). +> Source: `sdk/src/trace/span.h` and `sdk/include/opentelemetry/sdk/trace/span_data.h`. +> Queued spans release the wrapper, keeping only `SpanData` + attributes (~500 bytes). + +### 3.5.3 Memory Growth Characteristics + +```mermaid +--- +config: + xyChart: + width: 700 + height: 400 +--- +xychart-beta + title "Memory Usage vs Span Rate (bounded by queue limit)" + x-axis "Spans/second" [0, 200, 400, 600, 800, 1000] + y-axis "Memory (MB)" 0 --> 12 + line [8.5, 9.2, 9.6, 9.9, 10.0, 10.0] +``` + +**Notes**: + +- Memory increases with span rate but **plateaus at queue capacity** (default 2048 spans) +- Batch export prevents unbounded growth +- At queue limit, oldest spans are dropped (not blocked) +- Maximum memory is bounded: ~8.3 MB static (dominated by worker thread stack) + 2048 queued spans x ~500 bytes (~1 MB) + active spans (~0.8 MB) ≈ **~10 MB ceiling** +- The worker thread stack (~8 MB) is virtual memory; actual RSS depends on stack usage (typically much less) + +> **Measured outcome**: A perf-iac comparison (telemetry compiled-in + enabled vs compiled-out, +> 9 nodes — validators and client-handlers — under sustained payment load) recorded **no measurable +> RSS increase over the telemetry-off baseline** (~15 GiB mean / ~18–19 GiB peak on both sides), +> with no OOM, no swap, and no leak across the run. The ~10 MB ceiling above is therefore a +> provisioning safety margin (dominated by virtual thread-stack address space), not an expected +> resident-memory increase. Steady-state cost shows up as throughput (~3–4% at head sampling 1.0), +> not memory. + +### 3.5.4 Performance Data Sources + +The overhead estimates in Sections 3.3-3.5 are derived from the following sources: + +| Source | What it covers | URL | +| ------------------------------------------------ | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| OTel C++ SDK CI benchmarks (969 runs) | Span creation, context activation, sampler overhead | [Benchmark Dashboard](https://open-telemetry.github.io/opentelemetry-cpp/benchmarks/) | +| `api/test/trace/span_benchmark.cc` | API-level span creation (~22 ns no-op) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/api/test/trace/span_benchmark.cc) | +| `sdk/test/trace/sampler_benchmark.cc` | SDK span creation with samplers (~1,000 ns AlwaysOn) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/test/trace/sampler_benchmark.cc) | +| `sdk/include/.../span_data.h` | SpanData memory layout (~250 bytes base) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/include/opentelemetry/sdk/trace/span_data.h) | +| `sdk/src/trace/span.h` | Span wrapper memory layout (~88 bytes) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/src/trace/span.h) | +| `sdk/include/.../batch_span_processor_options.h` | Default queue size (2048), batch size (512) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/include/opentelemetry/sdk/trace/batch_span_processor_options.h) | +| `sdk/include/.../circular_buffer.h` | CircularBuffer implementation (AtomicUniquePtr array) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/include/opentelemetry/sdk/common/circular_buffer.h) | +| OTLP proto definition | Serialized span size estimation | [Proto](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto) | + +--- + +## 3.6 Network Overhead Analysis + +### 3.6.1 Export Bandwidth + +> **Bytes per span**: Estimates use ~500 bytes/span (conservative upper bound). OTLP protobuf analysis +> shows a typical span with 3-5 string attributes serializes to ~200-300 bytes raw; with gzip +> compression (~60-70% of raw) and batching (amortized headers), ~350 bytes/span is more realistic. +> The table uses the conservative estimate for capacity planning. + +| Sampling Rate | Spans/sec | Bandwidth | Notes | +| ------------- | --------- | --------- | ---------------- | +| 100% | ~500 | ~250 KB/s | Development only | +| 10% | ~50 | ~25 KB/s | Staging | +| 1% | ~5 | ~2.5 KB/s | Production | +| Error-only | ~1 | ~0.5 KB/s | Minimal overhead | + +### 3.6.2 Trace Context Propagation + +| Message Type | Context Size | Messages/sec | Overhead | +| ---------------------- | ------------ | ------------ | ----------- | +| TMTransaction | 25 bytes | ~100 | ~2.5 KB/s | +| TMProposeSet | 25 bytes | ~10 | ~250 B/s | +| TMValidation | 25 bytes | ~50 | ~1.25 KB/s | +| **Total P2P overhead** | | | **~4 KB/s** | + +--- + +## 3.7 Optimization Strategies + +### 3.7.1 Sampling Strategies + +#### Tail Sampling + +```mermaid +flowchart TD + trace["New Trace"] + + trace --> errors{"Is Error?"} + errors -->|Yes| sample["SAMPLE"] + errors -->|No| consensus{"Is Consensus?"} + + consensus -->|Yes| sample + consensus -->|No| slow{"Is Slow?"} + + slow -->|Yes| sample + slow -->|No| prob{"Random < 10%?"} + + prob -->|Yes| sample + prob -->|No| drop["DROP"] + + style sample fill:#4caf50,stroke:#388e3c,color:#fff + style drop fill:#f44336,stroke:#c62828,color:#fff +``` + +### 3.7.2 Batch Tuning Recommendations + +| Environment | Batch Size | Batch Delay | Max Queue | +| ------------------ | ---------- | ----------- | --------- | +| Low-latency | 128 | 1000ms | 512 | +| High-throughput | 1024 | 10000ms | 8192 | +| Memory-constrained | 256 | 2000ms | 512 | + +### 3.7.3 Conditional Instrumentation + +Instrumentation is gated on two levels. A compile-time feature flag (`XRPL_ENABLE_TELEMETRY`) reduces the trace macros to no-ops when telemetry is built out, so disabled builds carry zero cost. At runtime, per-component guards (e.g. `shouldTracePeer()`) skip span creation for components whose tracing is turned off, incurring no overhead beyond a single boolean check. + +--- + +## 3.8 Links to Detailed Documentation + +- **[Configuration Reference](./05-configuration-reference.md)**: Configuration options and collector setup +- **[Implementation Phases](./06-implementation-phases.md)**: Detailed timeline and milestones + +--- + +## 3.9 Code Intrusiveness Assessment + +> **TxQ** = Transaction Queue + +This section provides a detailed assessment of how intrusive the OpenTelemetry integration is to the existing xrpld codebase. + +### 3.9.1 Files Modified Summary + +| Component | Files Modified | Architectural Impact | +| --------------------- | -------------- | -------------------- | +| **Core Telemetry** | 10 new files | None (new module) | +| **Application Init** | 2 files | Minimal | +| **RPC Layer** | 3 files | Minimal | +| **Transaction Relay** | 4 files | Low | +| **Consensus** | 3 files | Low-Medium | +| **Protocol Buffers** | 1 file | Low | +| **CMake/Build** | 3 files | Minimal | +| **PathFinding** | 2 | Minimal | +| **TxQ/Fee** | 2 | Minimal | +| **Validator/Amend** | 3 | Minimal | +| **Total** | **~33 files** | **Low** | + +### 3.9.2 Detailed File Impact + +```mermaid +pie title Code Changes by Component + "New Telemetry Module" : 800 + "Transaction Relay" : 160 + "Consensus" : 130 + "RPC Layer" : 100 + "PathFinding" : 80 + "TxQ/Fee" : 60 + "Validator/Amendment" : 40 + "Application Init" : 35 + "Protocol Buffers" : 25 + "Build System" : 60 +``` + +#### New Files (No Impact on Existing Code) + +| File | Purpose | +| ------------------------------------------- | ------------------------- | +| `include/xrpl/telemetry/Telemetry.h` | Main interface | +| `include/xrpl/telemetry/TelemetryConfig.h` | Configuration structures | +| `include/xrpl/telemetry/TraceContext.h` | Context propagation | +| `include/xrpl/telemetry/SpanGuard.h` | RAII wrapper | +| `include/xrpl/telemetry/DiscardFlag.h` | Thread-local discard flag | +| `include/xrpl/telemetry/SpanAttributes.h` | Attribute helpers | +| `src/libxrpl/telemetry/Telemetry.cpp` | Implementation | +| `src/libxrpl/telemetry/TelemetryConfig.cpp` | Config parsing | +| `src/libxrpl/telemetry/TraceContext.cpp` | Context serialization | +| `src/libxrpl/telemetry/NullTelemetry.cpp` | No-op implementation | + +#### Modified Files (Existing Xrpld Code) + +| File | Risk Level | +| ------------------------------------------------- | ---------- | +| `src/xrpld/app/main/Application.cpp` | Low | +| `include/xrpl/core/ServiceRegistry.h` | Low | +| `src/xrpld/rpc/detail/ServerHandler.cpp` | Low | +| `src/xrpld/rpc/handlers/*.cpp` | Low | +| `src/xrpld/overlay/detail/PeerImp.cpp` | Medium | +| `src/xrpld/overlay/detail/OverlayImpl.cpp` | Medium | +| `src/xrpld/app/consensus/RCLConsensus.cpp` | Medium | +| `src/xrpld/app/consensus/RCLConsensusAdaptor.cpp` | Medium | +| `src/xrpld/core/JobQueue.cpp` | Low | +| `src/xrpld/app/paths/PathRequest.cpp` | Low | +| `src/xrpld/app/paths/Pathfinder.cpp` | Low | +| `src/xrpld/app/misc/TxQ.cpp` | Low | +| `src/xrpld/app/main/LoadManager.cpp` | Low | +| `src/xrpld/app/misc/ValidatorList.cpp` | Low | +| `src/xrpld/app/misc/AmendmentTable.cpp` | Low | +| `src/xrpld/app/misc/Manifest.cpp` | Low | +| `src/xrpld/shamap/SHAMap.cpp` | Low | +| `src/xrpld/overlay/detail/ripple.proto` | Low | +| `CMakeLists.txt` | Low | +| `cmake/FindOpenTelemetry.cmake` | None (new) | + +### 3.9.3 Risk Assessment by Component + +
+ +**Do First** ↖ ↗ **Plan Carefully** + +```mermaid +quadrantChart + title Code Intrusiveness Risk Matrix + x-axis Low Risk --> High Risk + y-axis Low Value --> High Value + + RPC Tracing: [0.2, 0.55] + Transaction Relay: [0.55, 0.85] + Consensus Tracing: [0.75, 0.92] + Peer Message Tracing: [0.85, 0.35] + JobQueue Context: [0.3, 0.42] + Ledger Acquisition: [0.48, 0.65] + PathFinding: [0.38, 0.72] + TxQ and Fees: [0.25, 0.62] + Validator Mgmt: [0.15, 0.35] +``` + +**Optional** ↙ ↘ **Avoid** + +
+ +#### Risk Level Definitions + +| Risk Level | Definition | Mitigation | +| ---------- | ---------------------------------------------------------------- | ---------------------------------- | +| **Low** | Additive changes only; no modification to existing logic | Standard code review | +| **Medium** | Minor modifications to existing functions; clear boundaries | Comprehensive unit tests | +| **High** | Changes to core logic or data structures; potential side effects | Integration tests + staged rollout | + +### 3.9.4 Architectural Impact Assessment + +| Aspect | Impact | Justification | +| -------------------- | ------- | -------------------------------------------------------------------------------- | +| **Data Flow** | Minimal | Read-only instrumentation; no modification to consensus or transaction data flow | +| **Threading Model** | Minimal | Context propagation uses thread-local storage (standard OTel pattern) | +| **Memory Model** | Low | Bounded queues prevent unbounded growth; RAII ensures cleanup | +| **Network Protocol** | Low | Optional fields in protobuf (high field numbers); backward compatible | +| **Configuration** | None | New config section; existing configs unaffected | +| **Build System** | Low | Optional CMake flag; builds work without OpenTelemetry | +| **Dependencies** | Low | OpenTelemetry SDK is optional; null implementation when disabled | + +### 3.9.5 Backward Compatibility + +| Compatibility | Status | Notes | +| --------------- | ------- | ----------------------------------------------------- | +| **Config File** | ✅ Full | New `[telemetry]` section is optional | +| **Protocol** | ✅ Full | Optional protobuf fields with high field numbers | +| **Build** | ✅ Full | `XRPL_ENABLE_TELEMETRY=OFF` produces identical binary | +| **Runtime** | ✅ Full | `enabled=0` produces zero overhead | +| **API** | ✅ Full | No changes to public RPC or P2P APIs | + +### 3.9.6 Rollback Strategy + +If issues are discovered after deployment: + +1. **Immediate**: Set `enabled=0` in config and restart (zero code change) +2. **Quick**: Rebuild with `XRPL_ENABLE_TELEMETRY=OFF` +3. **Complete**: Revert telemetry commits (clean separation makes this easy) + +### 3.9.7 Code Change Examples + +**Minimal RPC Instrumentation (Low Intrusiveness):** Instrumenting an RPC handler adds roughly 3-4 lines: one macro to start the span and one or two `setAttribute` calls (command name, status). The span ends automatically via RAII, so the existing control flow — process the request, send the result — is untouched. + +**Consensus Instrumentation (Medium Intrusiveness):** Consensus is slightly more intrusive because child spans in later phase transitions need the round's context. Beyond the span-start and attribute macros, this requires storing the active context in a new member variable (`currentRoundContext_`) at round start. The existing round logic itself remains unchanged. + +--- + +_Previous: [Design Decisions](./02-design-decisions.md)_ | _Next: [Configuration Reference](./05-configuration-reference.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/05-configuration-reference.md b/OpenTelemetryPlan/05-configuration-reference.md new file mode 100644 index 00000000000..7e47023f516 --- /dev/null +++ b/OpenTelemetryPlan/05-configuration-reference.md @@ -0,0 +1,275 @@ +# Configuration Reference + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Implementation Phases](./06-implementation-phases.md) + +--- + +## 5.1 xrpld Configuration + +> **OTLP** = OpenTelemetry Protocol | **TxQ** = Transaction Queue + +### 5.1.1 Configuration File Section + +The authoritative `[telemetry]` example lives in `cfg/xrpld-example.cfg`. Telemetry is disabled by default (`enabled=0`); enabling it turns on distributed tracing for transaction flow, consensus, and RPC calls, with traces exported to an OpenTelemetry Collector over OTLP. Head sampling is intentionally fixed at 1.0 (sample everything) and is not configurable — per-node head-sampling would produce broken/partial distributed traces, so volume reduction is delegated to the collector's tail sampling (see Section 7.4.2). The full option reference follows. + +### 5.1.2 Configuration Options Summary + +| Option | Type | Default | Description | +| -------------------------- | ------ | --------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `enabled` | bool | `false` | Enable/disable telemetry | +| `endpoint` | string | `http://localhost:4318/v1/traces` | OTLP/HTTP collector endpoint | +| `use_tls` | bool | `false` | Enable TLS for exporter connection | +| `tls_ca_cert` | string | `""` | Path to CA certificate file | +| `tls_client_cert` | string | `""` | Path to node's client certificate (PEM) for mutual TLS; requires `use_tls=1`; empty = one-way TLS | +| `tls_client_key` | string | `""` | Path to private key (PEM) for `tls_client_cert`; requires `use_tls=1`; required when the cert is set | +| `batch_size` | uint | `512` | Spans per export batch | +| `batch_delay_ms` | uint | `5000` | Max delay before sending batch (ms) | +| `max_queue_size` | uint | `2048` | Maximum queued spans | +| `trace_transactions` | bool | `true` | Enable transaction tracing | +| `trace_consensus` | bool | `true` | Enable consensus tracing | +| `trace_rpc` | bool | `true` | Enable RPC tracing | +| `trace_peer` | bool | `true` | Enable peer message tracing (high volume) | +| `trace_ledger` | bool | `true` | Enable ledger tracing | +| `tx_trace_strategy` | string | `"deterministic"` | TX trace ID strategy: `"deterministic"` (trace_id = txHash[0:16]) or `"attribute"` (random) | +| `consensus_trace_strategy` | string | `"deterministic"` | Consensus trace ID strategy: `"deterministic"` (trace_id = prevLedgerHash[0:16]) or `"attribute"` (random) | +| `service_name` | string | `"xrpld"` | Service name (`service.name`) for traces and metrics | +| `service_instance_id` | string | `` | Instance identifier | + +**Planned (not yet implemented)**: the following options appear in the design +documents but are not parsed by `TelemetryConfig.cpp` in Phase 1b and later +phases. They will be added as the corresponding subsystems are instrumented: + +| Option | Planned Phase | Purpose | +| -------------------------- | ------------- | ----------------------------------------------------------------------- | +| `exporter` | Future | Select between OTLP/HTTP and OTLP/gRPC | +| `trace_pathfind` | Phase 2 | Path computation tracing toggle | +| `trace_txq` | Phase 3 | Transaction queue tracing toggle | +| `trace_validator` | Future | Validator list / manifest update tracing | +| `trace_amendment` | Future | Amendment voting tracing | +| `consensus_trace_strategy` | Phase 4 | Trace ID strategy for consensus rounds (`deterministic` \| `attribute`) | + +--- + +## 5.2 Configuration Parser + +> **TxQ** = Transaction Queue + +The parser `setupTelemetry()` in `src/libxrpl/telemetry/TelemetryConfig.cpp` reads the `[telemetry]` `Section` and populates a `Telemetry::Setup` struct, applying the defaults listed in Section 5.1.2 via `section.value_or(...)`. It derives `serviceInstanceId` from the node public key when not overridden, selects the exporter endpoint default by exporter type, and leaves the sampling ratio at its fixed 1.0 default (not read from config — see Section 7.4.2). + +--- + +## 5.3 Application Integration + +### 5.3.1 ApplicationImp Changes + +> **Deferred identity**: The node public key (`nodeIdentity_`) is not +> available during `ApplicationImp`'s member initializer list — it is +> resolved later in `setup()`. The `Telemetry` object is therefore +> constructed with an empty `serviceInstanceId` and patched via +> `setServiceInstanceId()` once `setup()` has called `getNodeIdentity()`. + +`ApplicationImp` (in `src/xrpld/app/main/Application.cpp`) owns a `std::unique_ptr telemetry_`. It is built in the member initializer list via `makeTelemetry(setupTelemetry(...))` with an empty `serviceInstanceId`, then patched in `setup()` by calling `setServiceInstanceId()` with the Base58 node public key (unless the user supplied a custom `service_instance_id`). `start()` and `run()` forward to `telemetry_->start()` / `telemetry_->stop()`, and `getTelemetry()` returns the owned instance. + +### 5.3.2 ServiceRegistry Interface Addition + +`include/xrpl/core/ServiceRegistry.h` gains a pure-virtual `telemetry::Telemetry& getTelemetry()` (with a forward declaration of `telemetry::Telemetry`), giving every component a uniform accessor for the tracing subsystem. + +> **Note:** `Application` extends `ServiceRegistry`, so `getTelemetry()` is +> available on both. Components that hold a `ServiceRegistry&` (e.g. +> `NetworkOPsImp`) call `registry_.get().getTelemetry()`. Components that +> still hold an `Application&` (e.g. `ServerHandler`, `PeerImp`, +> `RCLConsensusAdaptor`) call `app_.getTelemetry()` directly. + +--- + +## 5.4 CMake Integration + +> **OTLP** = OpenTelemetry Protocol + +### 5.4.1 Find OpenTelemetry Module + +A `cmake/FindOpenTelemetry.cmake` module locates the OpenTelemetry C++ SDK. It first tries `find_package(opentelemetry-cpp CONFIG)`, aliasing the imported targets `OpenTelemetry::api`, `OpenTelemetry::sdk`, and `OpenTelemetry::otlp_grpc_exporter`, and falls back to `pkg-config` when no CMake config package is present. + +### 5.4.2 CMakeLists.txt Changes + +The top-level `CMakeLists.txt` adds an `XRPL_ENABLE_TELEMETRY` option (default `OFF`). When enabled, it runs `find_package(OpenTelemetry REQUIRED)`, defines the `XRPL_ENABLE_TELEMETRY` compile flag, and builds the `xrpl_telemetry` library from the real telemetry sources linked against the OpenTelemetry targets; when disabled, it builds the same target from a no-op `NullTelemetry.cpp` so call sites compile unchanged. + +--- + +## 5.5 OpenTelemetry Collector Configuration + +> **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring + +> **Production hardening**: The configurations in this section are starting points. For production deployments where xrpld ships telemetry across a network to a centrally-hosted collector, see [Securing the OTel Pipeline](./secure-OTel.md) for the required mTLS receiver config, NetworkPolicy, and peer trace-context validation. + +The authoritative collector config lives in the repo at `docker/telemetry/otel-collector-config.yaml` (with Tempo backend config in `docker/telemetry/tempo.yaml`). The sections below summarize the development and production shapes of that pipeline. + +### 5.5.1 Development Configuration + +The development collector enables an OTLP receiver on both gRPC (`0.0.0.0:4317`) and HTTP (`0.0.0.0:4318`), a single `batch` processor (1s timeout, batch size 100), and two exporters: a `logging` exporter for console debugging and `otlp/tempo` (insecure) for trace visualization. The single `traces` pipeline wires receiver → batch → both exporters. + +### 5.5.2 Production Configuration + +The production collector adds TLS on the OTLP gRPC receiver and a richer processor chain: a `memory_limiter` (OOM guard), `batch` (5s timeout, size 512), `tail_sampling`, and an `attributes` processor that hashes sensitive fields (e.g. `tx_account`) and stamps `deployment.environment`. Tail sampling keeps all `ERROR` traces, slow consensus rounds (>5s) and slow RPC requests (>1s), and probabilistically samples the remainder at 10%. Exporters target Grafana Tempo (TLS) and Elastic APM; `health_check` and `zpages` extensions are enabled for operability. + +--- + +## 5.6 Docker Compose Development Environment + +> **OTLP** = OpenTelemetry Protocol + +The authoritative development stack lives in the repo at `docker/telemetry/docker-compose.yml`. It brings up four services on a shared `xrpld-telemetry` network: an `otel-collector` (otel/opentelemetry-collector-contrib) exposing OTLP gRPC `4317`, OTLP HTTP `4318`, and health check `13133`; `tempo` for trace storage/visualization; `grafana` with provisioned datasources and dashboards (anonymous admin enabled); and an optional `prometheus` for metric correlation. + +--- + +## 5.7 Configuration Architecture + +> **OTLP** = OpenTelemetry Protocol + +```mermaid +flowchart TB + subgraph config["Configuration Sources"] + cfgFile["xrpld.cfg
[telemetry] section"] + cmake["CMake
XRPL_ENABLE_TELEMETRY"] + end + + subgraph init["Initialization"] + parse["setupTelemetry()"] + factory["makeTelemetry()"] + end + + subgraph runtime["Runtime Components"] + tracer["TracerProvider"] + exporter["OTLP Exporter"] + processor["BatchProcessor"] + end + + subgraph collector["Collector Pipeline"] + recv["Receivers"] + proc["Processors"] + exp["Exporters"] + end + + cfgFile --> parse + cmake -->|"compile flag"| parse + parse --> factory + factory --> tracer + tracer --> processor + processor --> exporter + exporter -->|"OTLP"| recv + recv --> proc + proc --> exp + + style config fill:#e3f2fd,stroke:#1976d2 + style runtime fill:#e8f5e9,stroke:#388e3c + style collector fill:#fff3e0,stroke:#ff9800 +``` + +**Reading the diagram:** + +- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, per-component trace toggles) while the CMake flag controls whether telemetry is compiled in at all. Head sampling is fixed at 1.0 and is not a config option; volume reduction happens via tail sampling in the collector. +- **Initialization**: `setupTelemetry()` parses config values, then `makeTelemetry()` constructs the provider, processor, and exporter objects. +- **Runtime Components**: The `TracerProvider` creates spans, the `BatchProcessor` buffers them, and the `OTLP Exporter` serializes and sends them over the wire. +- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP/HTTP and enters the external Collector pipeline. (OTLP/gRPC is future work — see design decisions §2.2.2.) +- **Collector Pipeline**: `Receivers` ingest OTLP data, `Processors` apply sampling/filtering/enrichment, and `Exporters` forward traces to storage backends (Tempo, etc.). + +--- + +## 5.8 Grafana Integration + +> **APM** = Application Performance Monitoring + +Step-by-step instructions for integrating xrpld traces with Grafana. + +### 5.8.1 Data Source Configuration + +#### Tempo (Recommended) + +A Tempo datasource (`grafana/provisioning/datasources/tempo.yaml`, provisioned from `docker/telemetry/grafana/`) points at `http://tempo:3200` and enables `tracesToLogs` (linking to Loki on `service.name`/`tx_hash` and mapping `trace_id` → `traceID`), `serviceMap` against Prometheus, the node graph, and Loki search. + +#### Elastic APM + +Alternatively, an Elasticsearch datasource (`grafana/provisioning/datasources/elastic-apm.yaml`) of type `elasticsearch` points at `http://elasticsearch:9200` against the `apm-*` index, using `@timestamp` as the time field and mapping the log message/level fields. + +### 5.8.2 Dashboard Provisioning + +A dashboard provider (`grafana/provisioning/dashboards/dashboards.yaml`) loads the `xrpld` dashboard folder from disk (`/var/lib/grafana/dashboards/rippled`), polling for changes every 30s with deletion disabled. + +### 5.8.3 Example Dashboard: RPC Performance + +An example `xrpld RPC Performance` dashboard (uid `xrpld-rpc-performance`) sourced from Tempo via TraceQL provides four panels: RPC latency by command (heatmap), RPC error rate by command (timeseries), the top 10 slowest RPC commands by average duration (table), and a recent-traces table. + +### 5.8.4 Example Dashboard: Transaction Tracing + +An example `xrpld Transaction Tracing` dashboard (uid `xrpld-tx-tracing`) over Tempo provides three panels: transaction throughput (`tx.receive` rate, stat), cross-node relay count (average `span.relay_count` on `tx.relay`, timeseries), and a table of transaction validation errors (`tx.validate` with `status.code=error`). + +### 5.8.5 TraceQL Query Examples + +Common queries for xrpld traces: + +``` +# Find all traces for a specific transaction hash +{resource.service.name="xrpld" && span.tx_hash="ABC123..."} + +# Find slow RPC commands (>100ms) +{resource.service.name="xrpld" && name=~"rpc.command.*"} | duration > 100ms + +# Find consensus rounds taking >5 seconds +{resource.service.name="xrpld" && name="consensus.round"} | duration > 5s + +# Find failed transactions with error details +{resource.service.name="xrpld" && name="tx.validate" && status.code=error} + +# Find transactions relayed to many peers +{resource.service.name="xrpld" && name="tx.relay"} | span.relay_count > 10 + +# Compare latency across nodes +{resource.service.name="xrpld" && name="rpc.command.account_info"} | avg(duration) by (resource.service.instance.id) +``` + +### 5.8.6 Correlation with PerfLog + +To correlate OpenTelemetry traces with existing PerfLog data: + +**Step 1: Configure Loki to ingest PerfLog** + +Configure a Promtail scrape job (`promtail-config.yaml`) that tails `/var/log/rippled/perf*.log`, parses each JSON line, and promotes `trace_id`, `ledger_seq`, and `tx_hash` to Loki labels. + +**Step 2: Add trace_id to PerfLog entries** + +Modify PerfLog so its JSON output includes a `trace_id` field whenever a valid span is active: fetch the current span from the OpenTelemetry runtime context, and if its context is valid, render the trace ID as a 32-character lowercase hex string into the log entry. + +**Step 3: Configure Grafana trace-to-logs link** + +In the Tempo datasource, set the `tracesToLogs` derived field to link to Loki on the `trace_id` and `tx_hash` tags, with `filterByTraceID: true`. + +### 5.8.7 Correlation with Insight/OTel System Metrics + +To correlate traces with Beast Insight system metrics: + +**Step 1: Export Insight metrics to Prometheus** + +Beast Insight metrics are exported natively via OTLP to the OTel Collector, +which exposes them on the Prometheus endpoint alongside spanmetrics. Configure +the `[insight]` section of `xrpld.cfg` with `server=otel`, +`endpoint=http://localhost:4318/v1/metrics`, and `prefix=xrpld`; no separate +StatsD exporter or Prometheus scrape job is needed when using `server=otel`. + +**Step 2: Add exemplars to metrics** + +The OpenTelemetry SDK automatically adds exemplars (trace IDs) to metrics when using the Prometheus exporter, linking metric spikes to specific traces. + +**Step 3: Configure Grafana metric-to-trace link** + +In the Prometheus datasource, set `exemplarTraceIdDestinations` to map the `trace_id` exemplar to the Tempo datasource. + +**Step 4: Dashboard panel with exemplars** + +Add a timeseries panel over Prometheus (e.g. `histogram_quantile(0.99, rate(rpc_duration_seconds_bucket[5m]))`) with `exemplar: true` enabled. + +This allows clicking on metric data points to jump directly to the related trace. + +--- + +_Previous: [Implementation Strategy](./03-implementation-strategy.md)_ | _Next: [Implementation Phases](./06-implementation-phases.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/06-implementation-phases.md b/OpenTelemetryPlan/06-implementation-phases.md new file mode 100644 index 00000000000..504b9d73639 --- /dev/null +++ b/OpenTelemetryPlan/06-implementation-phases.md @@ -0,0 +1,2208 @@ +# Implementation Phases + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Configuration Reference](./05-configuration-reference.md) | [Observability Backends](./07-observability-backends.md) + +--- + +## 6.1 Phase Overview + +> **TxQ** = Transaction Queue + +```mermaid +gantt + title OpenTelemetry Implementation Timeline + dateFormat YYYY-MM-DD + axisFormat Week %W + + section Phase 1 + Core Infrastructure :p1, 2024-01-01, 2w + SDK Integration :p1a, 2024-01-01, 4d + Telemetry Interface :p1b, after p1a, 3d + Configuration & CMake :p1c, after p1b, 3d + Unit Tests :p1d, after p1c, 2d + Buffer & Integration :p1e, after p1d, 2d + + section Phase 2 + RPC Tracing :p2, after p1, 2w + HTTP Context Extraction :p2a, after p1, 2d + RPC Handler Instrumentation :p2b, after p2a, 4d + PathFinding Instrumentation :p2f, after p2b, 2d + TxQ Instrumentation :p2g, after p2f, 2d + WebSocket Support :p2c, after p2g, 2d + Integration Tests :p2d, after p2c, 2d + Buffer & Review :p2e, after p2d, 4d + + section Phase 3 + Transaction Tracing :p3, after p2, 2w + Protocol Buffer Extension :p3a, after p2, 2d + PeerImp Instrumentation :p3b, after p3a, 3d + Fee Escalation Instrumentation :p3f, after p3b, 2d + Relay Context Propagation :p3c, after p3f, 3d + Multi-node Tests :p3d, after p3c, 2d + Buffer & Review :p3e, after p3d, 4d + + section Phase 4 + Consensus Tracing :p4, after p3, 2w + Consensus Round Spans :p4a, after p3, 3d + Proposal Handling :p4b, after p4a, 3d + Establish Phase (4a) :p4f, after p4b, 3d + Validation Tests :p4c, after p4f, 4d + Buffer & Review :p4e, after p4c, 4d + + section Phase 5 + Documentation & Deploy :p5, after p4, 1w + + section Phase 6 + StatsD Metrics Bridge :p6, after p5, 1w + + section Phase 7 + Native OTel Metrics :p7, after p6, 2w + + section Phase 8 + Log-Trace Correlation :p8, after p7, 1w + + section Phase 9 (Future) + Internal Metric Gap Fill :p9, after p8, 2.5w + + section Phase 10 (Future) + Workload Validation :p10, after p9, 2w + + section Phase 11 (Future) + Third-Party Collection :p11, after p10, 3w +``` + +--- + +## 6.2 Phase 1: Core Infrastructure (Weeks 1-2) + +**Objective**: Establish foundational telemetry infrastructure + +### Tasks + +| Task | Description | +| ---- | ----------------------------------------------------- | +| 1.1 | Add OpenTelemetry C++ SDK to Conan/CMake | +| 1.2 | Implement `Telemetry` interface and factory | +| 1.3 | Implement `SpanGuard` RAII wrapper | +| 1.4 | Implement configuration parser | +| 1.5 | Integrate into `ApplicationImp` | +| 1.6 | Add conditional compilation (`XRPL_ENABLE_TELEMETRY`) | +| 1.7 | Create `NullTelemetry` no-op implementation | +| 1.8 | Unit tests for core infrastructure | + +### Exit Criteria + +- [ ] OpenTelemetry SDK compiles and links +- [ ] Telemetry can be enabled/disabled via config +- [ ] Basic span creation works +- [ ] No performance regression when disabled +- [ ] Unit tests passing + +--- + +## 6.3 Phase 2: RPC Tracing (Weeks 3-4) + +> **TxQ** = Transaction Queue + +**Objective**: Complete tracing for all RPC operations + +### Tasks + +| Task | Description | +| ---- | -------------------------------------------------------------------------- | +| 2.1 | Implement W3C Trace Context HTTP header extraction | +| 2.2 | Instrument `ServerHandler::onRequest()` | +| 2.3 | Instrument `RPCHandler::doCommand()` | +| 2.4 | Add RPC-specific attributes | +| 2.5 | Instrument WebSocket handler | +| 2.6 | PathFinding instrumentation (`pathfind.request`, `pathfind.compute` spans) | +| 2.7 | TxQ instrumentation (`txq.enqueue`, `txq.apply` spans) | +| 2.8 | Integration tests for RPC tracing | +| 2.9 | Performance benchmarks | +| 2.10 | Documentation | + +### Exit Criteria + +- [ ] All RPC commands traced +- [ ] Trace context propagates from HTTP headers +- [ ] WebSocket and HTTP both instrumented +- [ ] <1ms overhead per RPC call +- [ ] Integration tests passing + +--- + +## 6.4 Phase 3: Transaction Tracing (Weeks 5-6) + +**Objective**: Trace transaction lifecycle across network with deterministic cross-node correlation + +### Tasks + +| Task | Description | +| ---- | -------------------------------------------------------------- | +| 3.1 | Define `TraceContext` Protocol Buffer message | +| 3.2 | Implement protobuf context serialization | +| 3.3 | Instrument `PeerImp::handleTransaction()` | +| 3.4 | Instrument `NetworkOPs::submitTransaction()` | +| 3.5 | Instrument HashRouter integration | +| 3.6 | Fee escalation instrumentation (`fee.escalate` span) | +| 3.7 | Implement relay context propagation | +| 3.8 | Integration tests (multi-node) | +| 3.9 | Deterministic transaction trace ID (`trace_id = txHash[0:16]`) | +| 3.10 | Performance benchmarks | + +### Deterministic Trace ID (Task 3.9) + +Transaction spans use **deterministic trace IDs** derived from the transaction hash: +`trace_id = txHash[0:16]`. All nodes handling the same transaction independently +produce spans under the same trace_id. Protobuf `span_id` propagation (Task 3.7) +additionally provides parent-child relay ordering when available. See +[02-design-decisions.md §2.5.0](./02-design-decisions.md) for the design rationale +and [Phase3_taskList.md Task 3.9](./Phase3_taskList.md) for the full implementation spec. + +### Exit Criteria + +- [ ] Transaction traces span across nodes +- [ ] Trace context in Protocol Buffer messages +- [ ] HashRouter deduplication visible in traces +- [ ] Multi-node integration tests passing +- [ ] <5% overhead on transaction throughput +- [ ] Deterministic trace_id: all nodes produce same trace_id for same transaction +- [ ] Protobuf span_id propagation preserves parent-child ordering when available + +--- + +## 6.5 Phase 4: Consensus Tracing (Weeks 7-8) + +**Objective**: Full observability into consensus rounds + +### Tasks + +| Task | Description | Status | +| ---- | ---------------------------------------------- | ------------------ | +| 4.1 | Instrument `RCLConsensusAdaptor::startRound()` | ✅ Done (via 4a.2) | +| 4.2 | Instrument phase transitions | ✅ Done | +| 4.3 | Instrument proposal handling | ✅ Done | +| 4.4 | Instrument validation handling | ✅ Done | +| 4.5 | Add consensus-specific attributes | ✅ Done | +| 4.6 | Correlate with transaction traces | ✅ Done | +| 4.7 | Build verification and testing | ✅ Done | +| 4.8 | Validation span enrichment (ext. dashboard) | ❌ Not done | + +**Note**: The original plan doc listed tasks 4.7-4.11 as "Validator list tracing", +"Amendment voting tracing", "SHAMap sync tracing", "Multi-validator integration tests", +and "Performance validation". These were descoped and replaced by the tasklist's 4.7 +(build verification) and 4.8 (validation span enrichment). Validator, amendment, and +SHAMap tracing are not implemented. + +### Spans Produced + +| Span Name | Location | Attributes | +| --------------------------- | ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `consensus.phase.open` | `Consensus.h` | _(none)_ | +| `consensus.proposal.send` | `RCLConsensus.cpp` | `consensus_round` | +| `consensus.ledger_close` | `RCLConsensus.cpp` | `ledger_seq`, `consensus_mode` | +| `consensus.accept` | `RCLConsensus.cpp` | `proposers`, `round_time_ms`, `quorum` | +| `consensus.accept.apply` | `RCLConsensus.cpp` | `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `ledger_seq`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction` | +| `consensus.validation.send` | `RCLConsensus.cpp` | `ledger_seq`, `proposing` | + +### Exit Criteria + +- [x] Complete consensus round traces +- [x] Phase transitions visible (open, establish, close, accept) +- [x] Proposals and validations traced — send and receive; relay deferred to Phase 4b +- [x] Close time agreement tracked (per `avCT_CONSENSUS_PCT`) +- [x] No impact on consensus timing +- [ ] Multi-validator test network validated +- [x] Transaction-consensus correlation (Task 4.6) — `tx.included` events in doAccept +- [ ] Validation span enrichment (Task 4.8) — not implemented + +### Implementation Status — Phase 4a Complete + +Phase 4a (establish-phase gap fill & cross-node correlation) adds: + +- **Deterministic trace ID** derived from `previousLedger.id()` so all validators + in the same round share the same `trace_id` (switchable via + `consensus_trace_strategy` config: `"deterministic"` or `"attribute"`). + See [Configuration Reference](./05-configuration-reference.md) for full + configuration options. +- **Round lifecycle spans**: `consensus.round` with round-to-round span links. +- **Establish phase**: `consensus.establish`, `consensus.update_positions` (with + `dispute.resolve` events), `consensus.check` (with threshold tracking). +- **Mode changes**: `consensus.mode_change` spans. +- **Validation**: `consensus.validation.send` with span link to round span + (thread-safe cross-thread access via `roundSpanContext_` snapshot). +- **Separation of concerns**: telemetry extracted to private helpers + (`startRoundTracing`, `createValidationSpan`, `startEstablishTracing`, + `updateEstablishTracing`, `endEstablishTracing`). + +See [Phase4_taskList.md](./Phase4_taskList.md) for the full spec and implementation notes. + +--- + +## 6.5a Phase 4a: Establish-Phase Gap Fill & Cross-Node Correlation + +**Objective**: Fill tracing gaps in the establish phase and establish cross-node +correlation using deterministic trace IDs derived from `previousLedger.id()`. + +**Approach**: Direct instrumentation in `Consensus.h` and `RCLConsensus.cpp`. +All spans use `SpanGuard` factory methods (`span()`, `hashSpan()`, `linkedSpan()`) +with `TraceCategory::Consensus` gating. No macros used — all tracing via direct +`SpanGuard` API calls. + +### Tasks + +| Task | Description | Effort | Risk | Status | +| ---- | ------------------------------------------------ | ------ | ------ | ------------------------ | +| 4a.0 | Prerequisites: extend SpanGuard & Telemetry APIs | 1d | Medium | ✅ Done (no macros) | +| 4a.1 | Adaptor `getTelemetry()` method | 0.5d | Low | ⏭️ Skipped (not needed) | +| 4a.2 | Switchable round span with deterministic traceID | 2d | High | ✅ Done | +| 4a.3 | Span members in `Consensus.h` | 0.5d | Medium | ✅ Done (with deviation) | +| 4a.4 | Instrument `phaseEstablish()` | 1d | Medium | ✅ Done | +| 4a.5 | Instrument `updateOurPositions()` | 1d | Medium | ✅ Done | +| 4a.6 | Instrument `haveConsensus()` (thresholds) | 1d | Medium | ✅ Done | +| 4a.7 | Instrument mode changes | 0.5d | Low | ✅ Done | +| 4a.8 | Reparent existing spans under round | 0.5d | Low | ✅ Done | +| 4a.9 | Build verification and testing | 1d | Low | ✅ Done | + +**Total Effort**: 9 days + +### Spans Produced + +| Span Name | Location | Key Attributes (actually set) | +| ---------------------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------- | +| `consensus.round` | `RCLConsensus.cpp` | `consensus_round_id`, `consensus_ledger_id`, `ledger_seq`, `consensus_mode`, `trace_strategy` | +| `consensus.establish` | `Consensus.h` | `converge_percent`, `establish_count`, `proposers` | +| `consensus.update_positions` | `Consensus.h` | `converge_percent`, `proposers`, `have_close_time_consensus`, `close_time_threshold`, `disputes_count`, `avalanche_threshold` | +| `consensus.check` | `Consensus.h` | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `consensus_result` | +| `consensus.mode_change` | `RCLConsensus.cpp` | `mode_old`, `mode_new` | + +### Exit Criteria + +- [x] Establish phase internals traced (establish, update_positions, check spans) +- [x] Establish phase fully traced — `disputes_count`, `avalanche_threshold`, dispute `yays`/`nays` all implemented +- [x] Cross-node correlation works via deterministic trace_id +- [x] Strategy switchable via config (`deterministic` / `attribute`) +- [x] Consecutive rounds linked via follows-from spans +- [x] Build passes with telemetry ON and OFF +- [x] No impact on consensus timing + +See [Phase4_taskList.md](./Phase4_taskList.md) for full task details. + +--- + +## 6.5b Phase 4b: Cross-Node Propagation (Future) + +**Objective**: Wire `TraceContextPropagator` for P2P messages (proposals, +validations) to enable true distributed tracing between nodes. + +**Status**: Partially implemented. Send-side injection (proposals and +validations) and receive-side extraction (`consensus.{proposal,validation}. +receive` spans parented on the sender's context) are wired in Phase 4a. +Remaining Phase 4b work: relay spans in `share(RCLCxPeerPos)` and multi-node +validation of the propagation path. + +**Prerequisites**: Phase 4a complete and validated. + +See [Phase4_taskList.md § Phase 4b](./Phase4_taskList.md) for full design. + +--- + +## 6.6 Phase 5: Documentation & Deployment (Week 9) + +**Objective**: Production readiness + +### Tasks + +| Task | Description | Status | +| ---- | ----------------------------- | ------------------- | +| 5.1 | Operator runbook | Complete | +| 5.2 | Grafana dashboards | Complete | +| 5.3 | Alert definitions | Deferred — post-MVP | +| 5.4 | Collector deployment examples | Complete | +| 5.5 | Developer documentation | Complete | +| 5.6 | Training materials | Deferred — post-MVP | +| 5.7 | Final integration testing | Complete | + +--- + +## 6.7 Phase 6: StatsD Metrics Integration (Week 10) + +**Objective**: Bridge xrpld's existing `beast::insight` StatsD metrics into the OpenTelemetry collection pipeline, exposing 300+ pre-existing metrics alongside span-derived RED metrics in Prometheus/Grafana. + +### Background + +xrpld has a mature metrics framework (`beast::insight`) that emits StatsD-format metrics over UDP. These metrics cover node health, peer networking, RPC performance, job queue, and overlay traffic — data that **does not** overlap with the span-based instrumentation from Phases 1-5. By adding a StatsD receiver to the OTel Collector, both metric sources converge in Prometheus. + +### Metric Inventory + +| Category | Group | Type | Count | Key Metrics | +| --------------- | ------------------ | ------------- | ---------- | ----------------------------------------------------------------------------------------------------------- | +| Node State | `State_Accounting` | Gauge | 10 | `*_duration`, `*_transitions` per operating mode | +| Ledger | `LedgerMaster` | Gauge | 2 | `Validated_Ledger_Age`, `Published_Ledger_Age` | +| Ledger Fetch | — | Counter | 1 | `ledger_fetches` | +| Ledger History | `ledger.history` | Counter | 1 | `mismatch` | +| RPC | `rpc` | Counter+Event | 3 | `requests`, `time` (histogram), `size` (histogram) | +| Job Queue | `jobq` | Gauge+Event | 1 + 2×N | `job_count`, per-job `{name}` and `{name}_q` (emitted with the `jobq_` group prefix, e.g. `jobq_job_count`) | +| Peer Finder | `Peer_Finder` | Gauge | 2 | `Active_Inbound_Peers`, `Active_Outbound_Peers` | +| Overlay | `Overlay` | Gauge | 1 | `Peer_Disconnects` | +| Overlay Traffic | per-category | Gauge | 4×57 = 228 | `Bytes_In/Out`, `Messages_In/Out` per traffic category | +| Pathfinding | — | Event | 2 | `pathfind_fast`, `pathfind_full` (histograms) | +| I/O | — | Event | 1 | `ios_latency` (histogram) | +| Resource Mgr | — | Meter | 2 | `warn`, `drop` (rate counters) | +| Caches | per-cache | Gauge | 2×N | `{cache}.size`, `{cache}.hit_rate` | + +**Total**: ~255+ unique metrics (plus dynamic job-type and cache metrics) + +### Tasks + +| Task | Description | +| ---- | --------------------------------------------------------------------------------------------------------------- | +| 6.1 | **DEFERRED** Fix Meter wire format (`\|m` → `\|c`) in StatsDCollector.cpp — breaking change, tracked separately | +| 6.2 | Add `statsd` receiver to OTel Collector config | +| 6.3 | Expose UDP port 8125 in docker-compose.yml | +| 6.4 | Add `[insight]` config to integration test node configs | +| 6.5 | Create "Node Health" Grafana dashboard (16 panels) | +| 6.6 | Create "Network Traffic" Grafana dashboard (10 panels) | +| 6.7 | Create "RPC & Pathfinding (StatsD)" Grafana dashboard (8 panels) | +| 6.8 | Update integration test to verify StatsD metrics in Prometheus | +| 6.9 | Update TESTING.md and telemetry-runbook.md | + +### Wire Format Fix (Task 6.1) — DEFERRED + +The `StatsDMeterImpl` in `StatsDCollector.cpp` sends metrics with `|m` suffix, which is non-standard StatsD. The OTel StatsD receiver silently drops these. Fix: change `|m` to `|c` (counter), which is semantically correct since meters are increment-only counters. Only 2 metrics are affected (`warn`, `drop` in Resource Manager). + +**Status**: Deferred as a separate change — this is a breaking change for any StatsD backend that previously consumed the custom `|m` type. The Resource Warnings and Resource Drops dashboard panels will show no data until this fix is applied. + +### New Grafana Dashboards + +**Node Health** (`statsd-node-health.json`, uid: `xrpld-statsd-node-health`): + +- Validated/Published Ledger Age, Operating Mode Duration/Transitions, I/O Latency, Job Queue Depth, Ledger Fetch Rate, Ledger History Mismatches, Key Jobs Execution/Dequeue Time, FullBelowCache Size/Hit Rate, Ledger Publish Gap, State Duration Rate, All Jobs Detail + +**Network Traffic** (`statsd-network-traffic.json`, uid: `xrpld-statsd-network`): + +- Active Inbound/Outbound Peers, Peer Disconnects, Total Bytes/Messages In/Out, Transaction/Proposal/Validation Traffic, Top Traffic Categories, Duplicate Traffic, All Traffic Categories Detail + +**RPC & Pathfinding (StatsD)** (`statsd-rpc-pathfinding.json`, uid: `xrpld-statsd-rpc`): + +- RPC Request Rate, Response Time p95/p50, Response Size p95/p50, Pathfinding Fast/Full Duration, Resource Warnings/Drops, Response Time Heatmap + +### Exit Criteria + +- [ ] StatsD metrics visible in Prometheus (`curl localhost:9090/api/v1/query?query=ledgermaster_validated_ledger_age`) +- [ ] All 3 new Grafana dashboards load without errors +- [ ] Integration test verifies at least core StatsD metrics (ledger age, peer counts, RPC requests) +- [ ] ~~Meter metrics (`warn`, `drop`) flow correctly after `|m` → `|c` fix~~ — DEFERRED (breaking change, tracked separately; resolved by Phase 7's OTel Counter mapping) + +--- + +## 6.8 Phase 7: Native OTel Metrics Migration (Weeks 11-12) + +**Objective**: Replace `StatsDCollector` with a native OpenTelemetry Metrics SDK implementation behind the existing `beast::insight::Collector` interface, eliminating the StatsD UDP dependency and unifying traces and metrics into a single OTLP pipeline. + +### Motivation: Why Migrate from StatsD to Native OTel Metrics + +The Phase 6 StatsD bridge was a pragmatic first step, but it retains inherent limitations that native OTel export resolves. + +#### What We Gain + +1. **Unified telemetry pipeline** — Traces and metrics export via the same OTLP/HTTP endpoint to the same OTel Collector. One protocol, one endpoint, one config. Eliminates the split-brain architecture of "OTLP for traces, StatsD UDP for metrics." + +2. **Eliminates StatsD UDP limitations** — StatsD is fire-and-forget over UDP with no delivery guarantees, no backpressure, 1472-byte MTU packet fragmentation, and text-based encoding overhead. OTLP uses HTTP/gRPC with retries, binary protobuf encoding, and connection-level flow control. + +3. **Fixes the `|m` wire format issue** — The `StatsDMeterImpl` uses non-standard `|m` StatsD type that the OTel StatsD receiver silently drops. Native OTel counters eliminate this problem entirely (Phase 6 Task 6.1 — DEFERRED becomes resolved). + +4. **Richer metric semantics** — OTel Metrics SDK supports explicit histogram bucket boundaries, exemplars (linking metrics to traces), resource attributes, and metric views. StatsD has no concept of these. + +5. **Removes infrastructure dependency** — No more StatsD receiver needed in the OTel Collector. One less receiver to configure, monitor, and debug. Simplifies the collector YAML. + +6. **Metric-to-trace correlation** — OTel metrics and traces share the same resource attributes (service.name, service.instance.id). Grafana can link from a metric spike directly to the traces that caused it — impossible with StatsD-sourced metrics. + +7. **Production-grade export** — OTel's `PeriodicMetricReader` provides configurable export intervals, batch sizes, timeout handling, and graceful shutdown — all built into the SDK rather than hand-rolled in `StatsDCollectorImp`. + +#### What We Lose + +1. **StatsD ecosystem compatibility** — Operators using external StatsD-compatible backends (Datadog Agent, Graphite, Telegraph) will need to switch to OTLP-compatible backends or keep `server=statsd` as a fallback. + +2. **Simplicity of UDP** — StatsD's UDP fire-and-forget model is dead simple and has zero connection management. OTLP/HTTP requires a TCP connection, TLS negotiation (in production), and retry logic. The OTel SDK handles this, but it's more moving parts. + +3. **Slightly higher memory** — OTel SDK maintains internal aggregation state for metrics before export. StatsD just formats and sends strings. Expected overhead: ~1-2 MB additional for metric state. + +4. **Dependency on OTel C++ Metrics SDK stability** — The Metrics SDK is GA since 1.0 and on version 1.18.0, but it's less battle-tested than the tracing SDK in the C++ ecosystem. + +#### Decision + +The gains (unified pipeline, delivery guarantees, metric-trace correlation, simpler collector config) significantly outweigh the losses. `StatsDCollector` is retained as a fallback via `server=statsd` for operators who need StatsD ecosystem compatibility during the transition period. + +### Architecture + +#### Class Hierarchy (after Phase 7) + +``` +beast::insight::Collector (abstract interface — unchanged) + | + +-- StatsDCollector (existing — retained as fallback, deprecated) + | +-- StatsDCounterImpl -> StatsD |c over UDP + | +-- StatsDGaugeImpl -> StatsD |g over UDP + | +-- StatsDMeterImpl -> StatsD |m over UDP (non-standard) + | +-- StatsDEventImpl -> StatsD |ms over UDP + | +-- StatsDHookImpl -> 1s periodic callback + | + +-- NullCollector (existing — unchanged, used when disabled) + | +-- NullCounterImpl -> no-op + | +-- NullGaugeImpl -> no-op + | +-- NullMeterImpl -> no-op + | +-- NullEventImpl -> no-op + | +-- NullHookImpl -> no-op + | + +-- OTelCollector (NEW — Phase 7) + +-- OTelCounterImpl -> otel::Counter + +-- OTelGaugeImpl -> otel::ObservableGauge + +-- OTelMeterImpl -> otel::Counter + +-- OTelEventImpl -> otel::Histogram + +-- OTelHookImpl -> 1s periodic callback (same pattern) +``` + +#### Data Flow (after Phase 7) + +```mermaid +graph LR + subgraph xrpldNode["xrpld Node"] + A["Trace Macros
XRPL_TRACE_SPAN"] + B["beast::insight
OTelCollector"] + end + + subgraph collector["OTel Collector :4317 / :4318"] + direction TB + R1["OTLP Receiver
:4317 gRPC | :4318 HTTP"] + BP["Batch Processor"] + SM["SpanMetrics Connector"] + + R1 --> BP + BP --> SM + end + + subgraph backends["Trace Backends"] + D["Tempo"] + end + + subgraph metrics["Metrics Stack"] + E["Prometheus :9090
scrapes :8889
span-derived + native OTel metrics"] + end + + subgraph viz["Visualization"] + F["Grafana :3000"] + end + + A -->|"OTLP/HTTP :4318
(traces)"| R1 + B -->|"OTLP/HTTP :4318
(metrics)"| R1 + + BP -->|"OTLP/gRPC"| D + SM -->|"RED metrics"| E + R1 -->|"system metrics
(native OTLP)"| E + + E --> F + D --> F + + style A fill:#4a90d9,color:#fff,stroke:#2a6db5 + style B fill:#d9534f,color:#fff,stroke:#b52d2d + style R1 fill:#5cb85c,color:#fff,stroke:#3d8b3d + style BP fill:#449d44,color:#fff,stroke:#2d6e2d + style SM fill:#449d44,color:#fff,stroke:#2d6e2d + style D fill:#f0ad4e,color:#000,stroke:#c78c2e + style E fill:#f0ad4e,color:#000,stroke:#c78c2e + style F fill:#5bc0de,color:#000,stroke:#3aa8c1 + style xrpldNode fill:#1a2633,color:#ccc,stroke:#4a90d9 + style collector fill:#1a3320,color:#ccc,stroke:#5cb85c + style backends fill:#332a1a,color:#ccc,stroke:#f0ad4e + style metrics fill:#332a1a,color:#ccc,stroke:#f0ad4e + style viz fill:#1a2d33,color:#ccc,stroke:#5bc0de +``` + +**Key change**: StatsD receiver removed from collector. Both traces and metrics enter via OTLP receiver on the same port. + +#### Configuration + +```ini +# [insight] section — new "otel" server option +[insight] +server=otel # NEW: uses OTel OTLP metrics exporter +prefix=xrpld # metric name prefix (preserved) + +# Endpoint and auth inherited from [telemetry] section: +[telemetry] +enabled=1 +endpoint=http://localhost:4318/v1/traces +``` + +The `OTelCollector` reads the OTLP endpoint from `[telemetry]` config (replacing `/v1/traces` with `/v1/metrics` for the metrics exporter). No additional config keys needed. + +**Backward compatibility**: `server=statsd` continues to work exactly as before. + +See [Phase7_taskList.md](./Phase7_taskList.md) for detailed per-task breakdown. + +### Instrument Type Mapping + +| beast::insight | OTel Metrics SDK | Rationale | +| ---------------------- | -------------------------------- | ---------------------------------------------------------------- | +| Counter (int64, `\|c`) | `Counter` | Direct 1:1 mapping | +| Gauge (uint64, `\|g`) | `ObservableGauge` | Async callback matches existing Hook polling pattern | +| Meter (uint64, `\|m`) | `Counter` | Fixes non-standard wire format; meters are semantically counters | +| Event (ms, `\|ms`) | `Histogram` | Duration distributions with explicit bucket boundaries | +| Hook (1s callback) | `PeriodicMetricReader` alignment | Same 1s collection interval | + +### Tasks + +| Task | Description | +| ---- | ------------------------------------------------------------------------- | +| 7.1 | Add OTel Metrics SDK to build deps (conan/cmake) | +| 7.2 | Implement `OTelCollector` class (~400-500 lines) | +| 7.3 | Update `CollectorManager` — add `server=otel` | +| 7.4 | Update OTel Collector YAML (add metrics pipeline, remove StatsD receiver) | +| 7.5 | Preserve metric names in Prometheus (naming strategy) | +| 7.6 | Update Grafana dashboards (if names change) | +| 7.7 | Update integration tests | +| 7.8 | Update documentation (runbook, reference docs) | + +### Exit Criteria + +- [ ] All 255+ metrics visible in Prometheus via OTLP pipeline (no StatsD receiver) +- [ ] `server=otel` is the default in development docker-compose +- [ ] `server=statsd` still works as a fallback +- [ ] Existing Grafana dashboards display data correctly +- [ ] Integration test passes with OTLP-only metrics pipeline +- [ ] No performance regression vs StatsD baseline (< 1% CPU overhead) +- [ ] Deferred Task 6.1 (`|m` wire format) no longer relevant + +--- + +## 6.8 Phase 7: Native OTel Metrics Migration (Weeks 11-12) + +**Objective**: Replace `StatsDCollector` with a native OpenTelemetry Metrics SDK implementation behind the existing `beast::insight::Collector` interface, eliminating the StatsD UDP dependency and unifying traces and metrics into a single OTLP pipeline. + +### Motivation: Why Migrate from StatsD to Native OTel Metrics + +The Phase 6 StatsD bridge was a pragmatic first step, but it retains inherent limitations that native OTel export resolves. + +#### What We Gain + +1. **Unified telemetry pipeline** — Traces and metrics export via the same OTLP/HTTP endpoint to the same OTel Collector. One protocol, one endpoint, one config. Eliminates the split-brain architecture of "OTLP for traces, StatsD UDP for metrics." + +2. **Eliminates StatsD UDP limitations** — StatsD is fire-and-forget over UDP with no delivery guarantees, no backpressure, 1472-byte MTU packet fragmentation, and text-based encoding overhead. OTLP uses HTTP/gRPC with retries, binary protobuf encoding, and connection-level flow control. + +3. **Fixes the `|m` wire format issue** — The `StatsDMeterImpl` uses non-standard `|m` StatsD type that the OTel StatsD receiver silently drops. Native OTel counters eliminate this problem entirely (Phase 6 Task 6.1 — DEFERRED becomes resolved). + +4. **Richer metric semantics** — OTel Metrics SDK supports explicit histogram bucket boundaries, exemplars (linking metrics to traces), resource attributes, and metric views. StatsD has no concept of these. + +5. **Removes infrastructure dependency** — No more StatsD receiver needed in the OTel Collector. One less receiver to configure, monitor, and debug. Simplifies the collector YAML. + +6. **Metric-to-trace correlation** — OTel metrics and traces share the same resource attributes (service.name, service.instance.id). Grafana can link from a metric spike directly to the traces that caused it — impossible with StatsD-sourced metrics. + +7. **Production-grade export** — OTel's `PeriodicMetricReader` provides configurable export intervals, batch sizes, timeout handling, and graceful shutdown — all built into the SDK rather than hand-rolled in `StatsDCollectorImp`. + +#### What We Lose + +1. **StatsD ecosystem compatibility** — Operators using external StatsD-compatible backends (Datadog Agent, Graphite, Telegraph) will need to switch to OTLP-compatible backends or keep `server=statsd` as a fallback. + +2. **Simplicity of UDP** — StatsD's UDP fire-and-forget model is dead simple and has zero connection management. OTLP/HTTP requires a TCP connection, TLS negotiation (in production), and retry logic. The OTel SDK handles this, but it's more moving parts. + +3. **Slightly higher memory** — OTel SDK maintains internal aggregation state for metrics before export. StatsD just formats and sends strings. Expected overhead: ~1-2 MB additional for metric state. + +4. **Dependency on OTel C++ Metrics SDK stability** — The Metrics SDK is GA since 1.0 and on version 1.18.0, but it's less battle-tested than the tracing SDK in the C++ ecosystem. + +#### Decision + +The gains (unified pipeline, delivery guarantees, metric-trace correlation, simpler collector config) significantly outweigh the losses. `StatsDCollector` is retained as a fallback via `server=statsd` for operators who need StatsD ecosystem compatibility during the transition period. + +### Architecture + +#### Class Hierarchy (after Phase 7) + +``` +beast::insight::Collector (abstract interface — unchanged) + | + +-- StatsDCollector (existing — retained as fallback, deprecated) + | +-- StatsDCounterImpl -> StatsD |c over UDP + | +-- StatsDGaugeImpl -> StatsD |g over UDP + | +-- StatsDMeterImpl -> StatsD |m over UDP (non-standard) + | +-- StatsDEventImpl -> StatsD |ms over UDP + | +-- StatsDHookImpl -> 1s periodic callback + | + +-- NullCollector (existing — unchanged, used when disabled) + | +-- NullCounterImpl -> no-op + | +-- NullGaugeImpl -> no-op + | +-- NullMeterImpl -> no-op + | +-- NullEventImpl -> no-op + | +-- NullHookImpl -> no-op + | + +-- OTelCollector (NEW — Phase 7) + +-- OTelCounterImpl -> otel::Counter + +-- OTelGaugeImpl -> otel::ObservableGauge + +-- OTelMeterImpl -> otel::Counter + +-- OTelEventImpl -> otel::Histogram + +-- OTelHookImpl -> 1s periodic callback (same pattern) +``` + +#### Data Flow (after Phase 7) + +```mermaid +graph LR + subgraph xrpldNode["xrpld Node"] + A["Trace Macros
XRPL_TRACE_SPAN"] + B["beast::insight
OTelCollector"] + end + + subgraph collector["OTel Collector :4317 / :4318"] + direction TB + R1["OTLP Receiver
:4317 gRPC | :4318 HTTP"] + BP["Batch Processor"] + SM["SpanMetrics Connector"] + + R1 --> BP + BP --> SM + end + + subgraph backends["Trace Backends"] + D["Jaeger / Tempo"] + end + + subgraph metrics["Metrics Stack"] + E["Prometheus :9090
scrapes :8889
span-derived + native OTel metrics"] + end + + subgraph viz["Visualization"] + F["Grafana :3000"] + end + + A -->|"OTLP/HTTP :4318
(traces)"| R1 + B -->|"OTLP/HTTP :4318
(metrics)"| R1 + + BP -->|"OTLP/gRPC"| D + SM -->|"RED metrics"| E + R1 -->|"system metrics
(native OTLP)"| E + + E --> F + D --> F + + style A fill:#4a90d9,color:#fff,stroke:#2a6db5 + style B fill:#d9534f,color:#fff,stroke:#b52d2d + style R1 fill:#5cb85c,color:#fff,stroke:#3d8b3d + style BP fill:#449d44,color:#fff,stroke:#2d6e2d + style SM fill:#449d44,color:#fff,stroke:#2d6e2d + style D fill:#f0ad4e,color:#000,stroke:#c78c2e + style E fill:#f0ad4e,color:#000,stroke:#c78c2e + style F fill:#5bc0de,color:#000,stroke:#3aa8c1 + style xrpldNode fill:#1a2633,color:#ccc,stroke:#4a90d9 + style collector fill:#1a3320,color:#ccc,stroke:#5cb85c + style backends fill:#332a1a,color:#ccc,stroke:#f0ad4e + style metrics fill:#332a1a,color:#ccc,stroke:#f0ad4e + style viz fill:#1a2d33,color:#ccc,stroke:#5bc0de +``` + +**Key change**: StatsD receiver removed from collector. Both traces and metrics enter via OTLP receiver on the same port. + +#### Configuration + +```ini +# [insight] section — new "otel" server option +[insight] +server=otel # NEW: uses OTel OTLP metrics exporter +prefix=xrpld # metric name prefix (preserved) + +# Endpoint and auth inherited from [telemetry] section: +[telemetry] +enabled=1 +endpoint=http://localhost:4318/v1/traces +``` + +The `OTelCollector` reads the OTLP endpoint from `[telemetry]` config (replacing `/v1/traces` with `/v1/metrics` for the metrics exporter). No additional config keys needed. + +**Backward compatibility**: `server=statsd` continues to work exactly as before. + +See [Phase7_taskList.md](./Phase7_taskList.md) for detailed per-task breakdown. + +### Instrument Type Mapping + +| beast::insight | OTel Metrics SDK | Rationale | +| ---------------------- | -------------------------------- | ---------------------------------------------------------------- | +| Counter (int64, `\|c`) | `Counter` | Direct 1:1 mapping | +| Gauge (uint64, `\|g`) | `ObservableGauge` | Async callback matches existing Hook polling pattern | +| Meter (uint64, `\|m`) | `Counter` | Fixes non-standard wire format; meters are semantically counters | +| Event (ms, `\|ms`) | `Histogram` | Duration distributions with explicit bucket boundaries | +| Hook (1s callback) | `PeriodicMetricReader` alignment | Same 1s collection interval | + +### Tasks + +| Task | Description | +| ---- | ------------------------------------------------------------------------- | +| 7.1 | Add OTel Metrics SDK to build deps (conan/cmake) | +| 7.2 | Implement `OTelCollector` class (~400-500 lines) | +| 7.3 | Update `CollectorManager` — add `server=otel` | +| 7.4 | Update OTel Collector YAML (add metrics pipeline, remove StatsD receiver) | +| 7.5 | Preserve metric names in Prometheus (naming strategy) | +| 7.6 | Update Grafana dashboards (if names change) | +| 7.7 | Update integration tests | +| 7.8 | Update documentation (runbook, reference docs) | + +### Exit Criteria + +- [ ] All 255+ metrics visible in Prometheus via OTLP pipeline (no StatsD receiver) +- [ ] `server=otel` is the default in development docker-compose +- [ ] `server=statsd` still works as a fallback +- [ ] Existing Grafana dashboards display data correctly +- [ ] Integration test passes with OTLP-only metrics pipeline +- [ ] No performance regression vs StatsD baseline (< 1% CPU overhead) +- [ ] Deferred Task 6.1 (`|m` wire format) no longer relevant + +--- + +## 6.9 Phase 8: Log-Trace Correlation and Centralized Log Ingestion (Week 13) + +### Motivation + +xrpld's `beast::Journal` logs and OpenTelemetry traces are currently two disjoint observability signals. When investigating an issue, operators must manually correlate timestamps between log files and Tempo traces. Phase 8 bridges this gap by injecting trace context (`trace_id`, `span_id`) into every log line emitted within an active span, and ingesting those logs into Grafana Loki via the OTel Collector's filelog receiver. + +#### Gains + +1. **One-click trace-to-log navigation** — Click a trace in Tempo and immediately see the corresponding log lines in Loki, filtered by `trace_id`. +2. **Reverse lookup (log-to-trace)** — Loki derived fields make `trace_id` values clickable links back to Tempo. +3. **Unified observability** — All three pillars (traces, metrics, logs) flow through the same OTel Collector pipeline and are visible in a single Grafana instance. +4. **Zero new dependencies in xrpld** — Uses existing OTel SDK headers (`GetSpan`, `GetContext`) already linked in Phase 1. +5. **Negligible overhead** — The implementation checks the thread-local context value directly, avoiding heap allocation on the no-span path (~15-20ns). On the active-span path, total cost is ~50ns per log call. At typical logging rates, overhead is negligible. + +#### Losses / Risks + +1. **Log format change** — Existing log parsers that rely on a fixed format will need updating to handle the optional `trace_id=... span_id=...` fields. +2. **Loki resource usage** — Log ingestion adds storage and memory overhead to the observability stack (mitigated by retention policies). +3. **Filelog receiver complexity** — The regex parser must be kept in sync with the log format; a format change in `Logs::format()` could break parsing. + +#### Decision + +The correlation value far outweighs the risks. The log format change is backward-compatible (fields are appended only when a span is active), and the filelog receiver regex is straightforward to maintain. + +### Architecture + +Phase 8 has two independent sub-phases that can be developed in parallel: + +- **Phase 8a (code change)**: Modify `Logs::format()` in `src/libxrpl/basics/Log.cpp` to append `trace_id= span_id=` when the current thread has an active OTel span. Guarded by `#ifdef XRPL_ENABLE_TELEMETRY`. +- **Phase 8b (infra only)**: Add Loki to the Docker Compose stack, configure the OTel Collector's `filelog` receiver to tail xrpld's log file, parse out structured fields (timestamp, partition, severity, trace_id, span_id, message), and export to Loki via OTLP. Configure Grafana Tempo↔Loki bidirectional linking. + +#### Trace ID Injection Flow + +```mermaid +flowchart LR + subgraph xrpld["xrpld process"] + JLOG["JLOG(j.info())"] + Format["Logs::format()"] + OTelCtx["OTel Context
(thread-local)"] + JLOG --> Format + OTelCtx -.->|"GetSpan()→GetContext()"| Format + end + + subgraph output["Log Output"] + LogLine["2024-01-15T10:30:45.123Z
LedgerMaster:NFO
trace_id=abc123...
span_id=def456...
Validated ledger 42"] + end + + Format --> LogLine + + style xrpld fill:#1a237e,stroke:#0d1642,color:#fff + style output fill:#1b5e20,stroke:#0d3d14,color:#fff + style JLOG fill:#283593,stroke:#1a237e,color:#fff + style Format fill:#283593,stroke:#1a237e,color:#fff + style OTelCtx fill:#283593,stroke:#1a237e,color:#fff + style LogLine fill:#2e7d32,stroke:#1b5e20,color:#fff +``` + +#### Loki Ingestion Pipeline + +```mermaid +flowchart LR + subgraph collector["OTel Collector"] + FR["filelog receiver
tails debug.log"] + RP["regex_parser
extracts trace_id,
span_id, severity"] + BP["batch processor"] + LE["otlp/loki exporter"] + FR --> RP --> BP --> LE + end + + LogFile["xrpld
debug.log"] --> FR + LE --> Loki["Grafana Loki
:3100"] + Loki <-->|"derivedFields ↔
tracesToLogs"| Tempo["Grafana Tempo"] + + style collector fill:#e65100,stroke:#bf360c,color:#fff + style FR fill:#f57c00,stroke:#e65100,color:#fff + style RP fill:#f57c00,stroke:#e65100,color:#fff + style BP fill:#f57c00,stroke:#e65100,color:#fff + style LE fill:#f57c00,stroke:#e65100,color:#fff + style LogFile fill:#1a237e,stroke:#0d1642,color:#fff + style Loki fill:#4a148c,stroke:#2e0d57,color:#fff + style Tempo fill:#4a148c,stroke:#2e0d57,color:#fff +``` + +### Tasks + +| Task | Description | +| ---- | ---------------------------------------------- | +| 8.1 | Inject trace_id into Logs::format() | +| 8.2 | Add Loki to Docker Compose stack | +| 8.3 | Add filelog receiver to OTel Collector | +| 8.4 | Configure Grafana trace-to-log correlation | +| 8.5 | Update integration tests | +| 8.6 | Update documentation (runbook, reference docs) | + +**Parallel work**: Task 8.2 (Loki infra) can run in parallel with Task 8.1 (code change). Tasks 8.3–8.6 are sequential. + +### Exit Criteria + +- [ ] Log lines within active spans contain `trace_id= span_id=` +- [ ] Log lines outside spans have no trace context (no empty fields) +- [ ] Loki ingests xrpld logs via OTel Collector filelog receiver +- [ ] Grafana Tempo → Loki one-click correlation works +- [ ] Grafana Loki → Tempo reverse lookup works via derived field +- [ ] Integration test verifies trace_id presence in logs +- [ ] No performance regression from trace_id injection (< 0.1% overhead) + +--- + +## 6.8.2 Phase 9: Internal Metric Instrumentation Gap Fill (Weeks 14-15) — Future Enhancement + +> **Status**: Planned, not yet implemented. + +### Motivation + +Phases 1-8 establish trace spans, StatsD metrics bridge, native OTel metrics, and log-trace correlation. However, ~68 metrics that exist inside xrpld's `get_counts`, `server_info`, TxQ, PerfLog, and `CountedObject` systems have **no time-series export path**. These are the metrics that exchanges, payment processors, analytics providers, validators, and researchers need most — NodeStore I/O performance, cache hit rates, per-RPC-method counters, transaction queue depth, fee escalation levels, and live object instance counts. + +### Architecture + +Hybrid approach — two instrumentation strategies based on proximity to existing code: + +```mermaid +flowchart TB + subgraph xrpld["xrpld process"] + subgraph existing["Existing beast::insight registrations"] + NS["NodeStore I/O
(Database.cpp)"] + end + subgraph newreg["New OTel MetricsRegistry"] + CR["Cache Hit Rates
(async gauge callbacks)"] + TQ["TxQ Metrics
(async gauge callbacks)"] + PL["PerfLog RPC/Job
(counters + histograms)"] + CO["CountedObjects
(async gauge callbacks)"] + LF["Load Factors
(async gauge callbacks)"] + end + end + + subgraph export["Export Pipelines"] + BI["beast::insight
OTelCollector (Phase 7)"] + OS["OTel Metrics SDK
PeriodicMetricReader"] + end + + NS --> BI + CR --> OS + TQ --> OS + PL --> OS + CO --> OS + LF --> OS + + BI --> OTLP["OTLP/HTTP :4318
/v1/metrics"] + OS --> OTLP + + style xrpld fill:#1a2633,color:#ccc,stroke:#4a90d9 + style existing fill:#2a4a6b,color:#fff,stroke:#4a90d9 + style newreg fill:#2a4a6b,color:#fff,stroke:#4a90d9 + style export fill:#1a3320,color:#ccc,stroke:#5cb85c + style NS fill:#4a90d9,color:#fff,stroke:#2a6db5 + style CR fill:#5cb85c,color:#fff,stroke:#3d8b3d + style TQ fill:#5cb85c,color:#fff,stroke:#3d8b3d + style PL fill:#5cb85c,color:#fff,stroke:#3d8b3d + style CO fill:#5cb85c,color:#fff,stroke:#3d8b3d + style LF fill:#5cb85c,color:#fff,stroke:#3d8b3d + style BI fill:#449d44,color:#fff,stroke:#2d6e2d + style OS fill:#449d44,color:#fff,stroke:#2d6e2d + style OTLP fill:#f0ad4e,color:#000,stroke:#c78c2e +``` + +- **beast::insight extensions** (blue): NodeStore I/O metrics added near existing `Database.cpp` registrations — exported via Phase 7's `OTelCollector`. +- **OTel MetricsRegistry** (green): New centralized class using `ObservableGauge` async callbacks for cache, TxQ, PerfLog, CountedObjects, and load factors — polled at 10s intervals by `PeriodicMetricReader`. + +### Third-Party Consumer Context + +| Consumer Category | Key Metrics They Need From Phase 9 | +| ---------------------- | --------------------------------------------------------------- | +| Exchanges | Fee escalation levels, TxQ depth, settlement latency | +| Payment Processors | Load factors, io_latency, transaction throughput | +| Analytics Providers | NodeStore I/O, cache hit rates, counted objects | +| Validators / Operators | Per-job execution times, PerfLog RPC counters, consensus timing | +| Academic Researchers | Consensus performance time-series, fee market dynamics | +| Institutional Custody | Server health scores, reserve calculations, node availability | + +### Tasks + +| Task | Description | +| ---- | ----------------------------------------- | +| 9.1 | NodeStore I/O metrics | +| 9.2 | Cache hit rate metrics + MetricsRegistry | +| 9.3 | TxQ metrics | +| 9.4 | PerfLog per-RPC metrics | +| 9.5 | PerfLog per-job metrics | +| 9.6 | Counted object instance metrics | +| 9.7 | Fee escalation & load factor metrics | +| 9.7a | push_metrics.py parity gauges | +| 9.8 | New Grafana dashboards (2 new, 2 updated) | +| 9.9 | Update documentation | +| 9.10 | Integration tests | + +See [Phase9_taskList.md](./Phase9_taskList.md) for detailed per-task breakdown. + +### Exit Criteria + +- [ ] All ~68 new metrics visible in Prometheus via OTLP pipeline +- [ ] `MetricsRegistry` class registers/deregisters cleanly with OTel SDK +- [ ] 2 new Grafana dashboards operational (Fee Market, Job Queue) +- [ ] No performance regression (< 0.5% CPU overhead from new callbacks) +- [ ] Documentation updated with full new metric inventory + +--- + +## 6.8.3 Phase 10: Synthetic Workload Generation & Telemetry Validation (Weeks 16-17) + +> **Status**: In progress. + +### Motivation + +Before the telemetry stack (Phases 1-9) can be considered production-ready, we need automated proof that all spans, attributes, metrics, Grafana dashboards, and log-trace correlation work correctly under realistic load. This phase establishes a reusable CI-integrated validation suite and performance benchmark baseline. + +### Architecture + +The validation uses a **2-node** validator cluster running as local processes alongside a Docker Compose telemetry stack (Collector, Tempo, Prometheus, Grafana). Two nodes are sufficient for consensus rounds and peer-to-peer span validation while minimizing CI resource usage. + +```mermaid +flowchart LR + subgraph harness["2-Node Validator Cluster (local processes)"] + direction TB + V1["Validator 1"] ~~~ V2["Validator 2"] + end + + subgraph telemetry["Docker Compose Telemetry Stack"] + direction TB + COL["OTel Collector
(OTLP + StatsD)"] + JAE["Tempo
(trace search)"] + PROM["Prometheus
(metrics)"] + GRAF["Grafana
(dashboards)"] + end + + subgraph generators["Workload Generators"] + RPC["RPC Load Generator
(configurable RPS,
command distribution)"] + TX["Transaction Submitter
(10 tx types via
WebSocket command API)"] + end + + subgraph validation["Validation Suite"] + SV["Span Validator
(Tempo API)"] + MV["Metric Validator
(Prometheus API,
all 26 metrics required)"] + DV["Dashboard Validator
(Grafana API)"] + BM["Benchmark Suite
(CPU, memory, latency
ON vs OFF comparison)"] + end + + generators --> harness + harness --> telemetry + telemetry --> validation + + style harness fill:#1a2633,color:#ccc,stroke:#4a90d9 + style telemetry fill:#1a2633,color:#ccc,stroke:#4a90d9 + style generators fill:#1a3320,color:#ccc,stroke:#5cb85c + style validation fill:#332a1a,color:#ccc,stroke:#f0ad4e + style V1 fill:#4a90d9,color:#fff,stroke:#2a6db5 + style V2 fill:#4a90d9,color:#fff,stroke:#2a6db5 + style COL fill:#4a90d9,color:#fff,stroke:#2a6db5 + style JAE fill:#4a90d9,color:#fff,stroke:#2a6db5 + style PROM fill:#4a90d9,color:#fff,stroke:#2a6db5 + style GRAF fill:#4a90d9,color:#fff,stroke:#2a6db5 + style RPC fill:#5cb85c,color:#fff,stroke:#3d8b3d + style TX fill:#5cb85c,color:#fff,stroke:#3d8b3d + style SV fill:#f0ad4e,color:#000,stroke:#c78c2e + style MV fill:#f0ad4e,color:#000,stroke:#c78c2e + style DV fill:#f0ad4e,color:#000,stroke:#c78c2e + style BM fill:#f0ad4e,color:#000,stroke:#c78c2e +``` + +### Key Implementation Details + +- **Transaction submitter and RPC load generator** both use xrpld's native WebSocket command format (`{"command": ...}`) — not JSON-RPC format. Response data lives inside `"result"` with `"status"` at the top level. +- **Node config** requires `[signing_support] true` for server-side signing, and `[ips]` (not `[ips_fixed]`) to ensure peer connections count in `peer_finder_active_*` metrics. +- **Metric validation** uses the Prometheus `/api/v1/series` endpoint (not instant queries) to avoid false negatives from stale StatsD gauges. Every metric in `expected_metrics.json` must have > 0 series. +- **StatsD gauge fix**: `StatsDGaugeImpl` initializes `m_dirty = true` so all gauges emit their initial value on first flush. Without this, gauges starting at 0 that never change (e.g. `jobq_job_count`) would be invisible in Prometheus. +- **I/O latency fix**: `io_latency_sampler` emits unconditionally on first sample, then applies the 10 ms threshold. This ensures `ios_latency` is registered in Prometheus even in low-load CI environments. +- **tx.receive span**: Sets default attributes (`xrpl.tx.suppressed = false`, `xrpl.tx.status = "new"`) on span creation so they are always present. The suppressed/bad code paths override these when applicable. + +### Tasks + +| Task | Description | +| ---- | -------------------------------------- | +| 10.1 | Multi-node test harness (5 validators) | +| 10.2 | RPC load generator | +| 10.3 | Transaction submitter (6+ tx types) | +| 10.4 | Telemetry validation suite | +| 10.5 | Performance benchmark suite | +| 10.6 | CI integration | +| 10.7 | Documentation | + +See [Phase10_taskList.md](./Phase10_taskList.md) for detailed per-task breakdown. + +### Validation Check Inventory (71 Checks) + +The validation suite (`validate_telemetry.py`) runs exactly 71 checks, broken down as: + +- **1 service registration** — `xrpld` exists in Tempo +- **17 span existence** — `rpc.request`, `rpc.process`, `rpc.ws_message`, `rpc.command.*`, `tx.process`, `tx.receive`, `tx.apply`, `consensus.proposal.send`, `consensus.ledger_close`, `consensus.accept`, `consensus.validation.send`, `consensus.accept.apply`, `ledger.build`, `ledger.validate`, `ledger.store`, `peer.proposal.receive`, `peer.validation.receive` +- **14 span attribute** — required attributes on the 14 spans that define them (22 unique attributes total) +- **2 span hierarchies** — `rpc.process` -> `rpc.command.*`, `ledger.build` -> `tx.apply` (1 skipped: `rpc.request` -> `rpc.process`, cross-thread) +- **1 span duration bounds** — all spans > 0 and < 60 s +- **26 metric existence** — 4 SpanMetrics (`span_calls_total`, `span_duration_milliseconds_{bucket,count,sum}`), 6 StatsD gauges (`ledgermaster_validated_ledger_age`, `published_ledger_age`, `state_accounting_full_duration`, `peer_finder_active_{inbound,outbound}_peers`, `jobq_job_count`), 2 StatsD counters (`rpc_requests_total`, `ledger_fetches_total`), 3 StatsD histograms (`rpc_time`, `rpc_size`, `ios_latency`), 4 overlay traffic (`total_bytes_{in,out}`, `total_messages_{in,out}`), 7 Phase 9 OTLP (`nodestore_state`, `cache_metrics`, `txq_metrics`, `rpc_method_{started,finished}_total`, `object_count`, `load_factor_metrics`) +- **10 dashboard loads** — `rpc-performance`, `transaction-overview`, `consensus-health`, `ledger-operations`, `peer-network`, `node-health`, `network-traffic`, `rpc-pathfinding`, `overlay-traffic-detail`, `ledger-data-sync` + +See [Phase10_taskList.md](./Phase10_taskList.md) for the full numbered check-by-check enumeration. + +### Current Status + +**Working** (71/71 checks pass in CI): +All 17 spans, 26 metrics, 10 dashboards, 14 attribute checks, 2 hierarchies, and duration bounds validated. + +**Not implemented or not available in CI**: + +1. `rpc.request` -> `rpc.process` parent-child hierarchy — skipped (cross-thread context propagation) +2. Log-trace correlation validation (Loki) — not included in checks +3. Full 255+ StatsD metric coverage — only 26 representative metrics validated +4. Sustained load / backpressure testing — not implemented +5. `docs/telemetry-runbook.md` updates — not done +6. `09-data-collection-reference.md` "Validation" section — not done +7. **Automated cross-CI baseline persistence** — the regression gate reads a + committed baseline; baseline updates flow through a manual PR refresh, not + an artifact promoted from `develop` (FU-2). + +### Exit Criteria + +- [x] 2-node validator cluster starts and reaches consensus +- [x] Validation suite confirms all required spans, attributes, and metrics (71/71 checks) +- [x] All 10 Grafana dashboards render data +- [ ] Benchmark shows < 3% CPU overhead, < 5MB memory overhead +- [x] CI workflow runs validation on telemetry branch changes +- [x] OTel-driven regression gate: captures per-span/per-RPC/per-job timings + from Prometheus and compares against a committed baseline + +--- + +## 6.8.4 Phase 11: Third-Party Data Collection Pipelines (Weeks 18-20) — Future Enhancement + +> **Status**: Planned, not yet implemented. + +### Motivation + +xrpld has no native Prometheus/OTLP metrics export for data accessible only via JSON-RPC (`server_info`, `get_counts`, `fee`, `peers`, `validators`, `feature`). Every external consumer — exchanges, payment processors, analytics providers, validators, compliance firms, DeFi protocols, researchers, custodians, and CBDC platforms — must build custom JSON-RPC polling and conversion pipelines. This phase centralizes that work into a reusable custom OTel Collector receiver. + +### Architecture + +```mermaid +flowchart LR + subgraph receiver["Custom OTel Collector Receiver (Go)"] + direction TB + SI["server_info
collector"] + GC["get_counts
collector"] + FE["fee
collector"] + PE["peers
collector"] + VA["validators
collector"] + DX["DEX/AMM
collector
(optional)"] + end + + xrpld["xrpld
Admin RPC
:5005"] -->|"JSON-RPC
poll every 30s"| receiver + + receiver -->|"xrpl_* metrics"| PROM["Prometheus
:9090"] + receiver -->|"OTLP export"| OTLP["Any OTLP-
compatible
backend"] + + PROM --> GF["Grafana
4 new dashboards"] + PROM --> AL["Prometheus
Alerting Rules"] + + style receiver fill:#1a3320,color:#ccc,stroke:#5cb85c + style SI fill:#5cb85c,color:#fff,stroke:#3d8b3d + style GC fill:#5cb85c,color:#fff,stroke:#3d8b3d + style FE fill:#5cb85c,color:#fff,stroke:#3d8b3d + style PE fill:#5cb85c,color:#fff,stroke:#3d8b3d + style VA fill:#5cb85c,color:#fff,stroke:#3d8b3d + style DX fill:#449d44,color:#fff,stroke:#2d6e2d + style xrpld fill:#4a90d9,color:#fff,stroke:#2a6db5 + style PROM fill:#f0ad4e,color:#000,stroke:#c78c2e + style OTLP fill:#f0ad4e,color:#000,stroke:#c78c2e + style GF fill:#5bc0de,color:#000,stroke:#3aa8c1 + style AL fill:#d9534f,color:#fff,stroke:#b52d2d +``` + +### Third-Party Consumer Gap Analysis + +| Consumer Category | Data Unlocked by Phase 11 | +| ---------------------- | ------------------------------------------------------------ | +| Exchanges | Real-time fee estimates, TxQ capacity, server health scores | +| Payment Processors | Settlement latency percentiles, corridor health | +| Analytics Providers | Validator metrics, network topology, amendment voting status | +| DeFi / AMM | AMM pool TVL, DEX order book depth, trade volumes | +| Validators / Operators | Per-peer latency, version distribution, UNL health, alerting | +| Compliance | Transaction volume trends, network growth metrics | +| Academic Researchers | Consensus performance time-series, decentralization metrics | +| CBDC / Tokenization | Token supply tracking, trust line adoption, freeze status | +| Institutional Custody | Multi-sig status, escrow tracking, reserve calculations | +| Wallet Providers | Server health for node selection, fee prediction data | + +### Tasks + +| Task | Description | +| ----- | ------------------------------------- | +| 11.1 | OTel Collector receiver scaffold (Go) | +| 11.2 | server_info / server_state collector | +| 11.3 | get_counts collector | +| 11.4 | Peer topology collector | +| 11.5 | Validator & amendment collector | +| 11.6 | Fee & TxQ collector | +| 11.7 | DEX & AMM collector (optional) | +| 11.8 | Prometheus alerting rules | +| 11.9 | New Grafana dashboards (4) | +| 11.10 | Integration with Phase 10 validation | +| 11.11 | Documentation | + +See [Phase11_taskList.md](./Phase11_taskList.md) for detailed per-task breakdown. + +### Exit Criteria + +- [ ] Custom OTel Collector receiver exports all `xrpl_*` metrics to Prometheus +- [ ] 4 new Grafana dashboards operational (Validator Health, Network Topology, Fee Market, DEX/AMM) +- [ ] Prometheus alerting rules fire correctly for simulated failures +- [ ] Receiver handles xrpld restart/unavailability gracefully +- [ ] Go receiver has unit tests with >80% coverage + +--- + +## 6.9 Risk Assessment + +```mermaid +quadrantChart + title Risk Assessment Matrix + x-axis Low Impact --> High Impact + y-axis Low Likelihood --> High Likelihood + quadrant-1 Mitigate Immediately + quadrant-2 Plan Mitigation + quadrant-3 Accept Risk + quadrant-4 Monitor Closely + + SDK Compat: [0.2, 0.18] + Protocol Chg: [0.75, 0.72] + Perf Overhead: [0.58, 0.42] + Context Prop: [0.4, 0.55] + Memory Leaks: [0.85, 0.25] +``` + +### Risk Details + +| Risk | Likelihood | Impact | Mitigation | +| ------------------------------------ | ---------- | ------ | --------------------------------------- | +| Protocol changes break compatibility | Medium | High | Use high field numbers, optional fields | +| Performance overhead unacceptable | Medium | Medium | Sampling, conditional compilation | +| Context propagation complexity | Medium | Medium | Phased rollout, extensive testing | +| SDK compatibility issues | Low | Medium | Pin SDK version, fallback to no-op | +| Memory leaks in long-running nodes | Low | High | Memory profiling, bounded queues | + +--- + +## 6.10 Success Metrics + +| Metric | Target | Measurement | +| ------------------------ | -------------------------------------------------------------- | --------------------- | +| Trace coverage | >95% of transaction code paths (independent of sampling ratio) | Sampling verification | +| CPU overhead | <3% | Benchmark tests | +| Memory overhead | <10 MB | Memory profiling | +| Latency impact (p99) | <2% | Performance tests | +| Trace completeness | >99% spans with required attrs | Validation script | +| Cross-node trace linkage | >90% of multi-hop transactions | Integration tests | + +--- + +## 6.11 Quick Wins and Crawl-Walk-Run Strategy + +> **TxQ** = Transaction Queue + +This section outlines a prioritized approach to maximize ROI with minimal initial investment. + +### 6.11.1 Crawl-Walk-Run Overview + +
+ +```mermaid +flowchart TB + subgraph crawl["🐢 CRAWL (Week 1-2)"] + direction LR + c1[Core SDK Setup] ~~~ c2[RPC Tracing Only] ~~~ c3[PathFinding + TxQ Tracing] ~~~ c4[Single Node] + end + + subgraph walk["🚶 WALK (Week 3-5)"] + direction LR + w1[Transaction Tracing] ~~~ w2[Fee Escalation Tracing] ~~~ w3[Cross-Node Context] ~~~ w4[Basic Dashboards] + end + + subgraph run["🏃 RUN (Week 6-9)"] + direction LR + r1[Consensus Tracing] ~~~ r2[Establish Phase
& Cross-Node Correlation] ~~~ r3[StatsD Integration] ~~~ r4[Production Deploy] + end + + crawl --> walk --> run + + style crawl fill:#1b5e20,stroke:#0d3d14,color:#fff + style walk fill:#bf360c,stroke:#8c2809,color:#fff + style run fill:#0d47a1,stroke:#082f6a,color:#fff + style c1 fill:#1b5e20,stroke:#0d3d14,color:#fff + style c2 fill:#1b5e20,stroke:#0d3d14,color:#fff + style c3 fill:#1b5e20,stroke:#0d3d14,color:#fff + style c4 fill:#1b5e20,stroke:#0d3d14,color:#fff + style w1 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style w2 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style w3 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style w4 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style r1 fill:#0d47a1,stroke:#082f6a,color:#fff + style r2 fill:#0d47a1,stroke:#082f6a,color:#fff + style r3 fill:#0d47a1,stroke:#082f6a,color:#fff + style r4 fill:#0d47a1,stroke:#082f6a,color:#fff +``` + +
+ +**Reading the diagram:** + +- **CRAWL (Weeks 1-2)**: Minimal investment -- set up the SDK, instrument RPC and PathFinding/TxQ handlers, and verify on a single node. Delivers immediate latency visibility. +- **WALK (Weeks 3-5)**: Expand to transaction lifecycle tracing, fee escalation, cross-node context propagation, and basic Grafana dashboards. This is where distributed tracing starts working. +- **RUN (Weeks 6-9)**: Full consensus instrumentation, establish-phase gap fill, cross-node correlation, StatsD integration, and production deployment with sampling and alerting. +- **Arrows (crawl → walk → run)**: Each phase builds on the prior one; you cannot skip ahead because later phases depend on infrastructure established earlier. + +### 6.11.2 Quick Wins (Immediate Value) + +| Quick Win | Value | When to Deploy | +| ------------------------------ | ------ | -------------- | +| **RPC Command Tracing** | High | Week 2 | +| **RPC Latency Histograms** | High | Week 2 | +| **Error Rate Dashboard** | Medium | Week 2 | +| **Transaction Submit Tracing** | High | Week 3 | +| **Consensus Round Duration** | Medium | Week 6 | + +### 6.11.3 CRAWL Phase (Weeks 1-2) + +**Goal**: Get basic tracing working with minimal code changes. + +**What You Get**: + +- RPC request/response traces for all commands +- Latency breakdown per RPC command +- PathFinding and TxQ tracing (directly impacts RPC latency) +- Error visibility with stack traces +- Basic Grafana dashboard + +**Code Changes**: ~15 lines in `ServerHandler.cpp`, ~40 lines in new telemetry module + +**Why Start Here**: + +- RPC is the lowest-risk, highest-visibility component +- PathFinding and TxQ are RPC-adjacent and directly affect latency +- Immediate value for debugging client issues +- No cross-node complexity +- Single file modification to existing code + +### 6.11.4 WALK Phase (Weeks 3-5) + +**Goal**: Add transaction lifecycle tracing across nodes. + +**What You Get**: + +- End-to-end transaction traces from submit to relay +- Fee escalation tracing within the transaction pipeline +- Cross-node correlation (see transaction path) +- HashRouter deduplication visibility +- Relay latency metrics + +**Code Changes**: ~120 lines across 4 files, plus protobuf extension + +**Why Do This Second**: + +- Builds on RPC tracing (transactions submitted via RPC) +- Fee escalation is integral to the transaction processing pipeline +- Moderate complexity (requires context propagation) +- High value for debugging transaction issues + +### 6.11.5 RUN Phase (Weeks 6-9) + +**Goal**: Full observability including consensus. + +**What You Get**: + +- Complete consensus round visibility +- Phase transition timing +- Validator proposal tracking +- ~~Validator list and manifest tracing~~ — descoped +- ~~Amendment voting tracing~~ — descoped +- ~~SHAMap sync tracing~~ — descoped +- Full end-to-end traces (client → RPC → TX → consensus → ledger) — partial (tx-consensus correlation not yet done) + +**Code Changes**: ~100 lines across 3 consensus files + +**Why Do This Last**: + +- Highest complexity (consensus is critical path) +- Validator, amendment, and SHAMap components were descoped (lower priority) +- Requires thorough testing +- Lower relative value (consensus issues are rarer) + +### 6.11.6 ROI Prioritization Matrix + +```mermaid +quadrantChart + title Implementation ROI Matrix + x-axis Low Effort --> High Effort + y-axis Low Value --> High Value + quadrant-1 Quick Wins - Do First + quadrant-2 Major Projects - Plan Carefully + quadrant-3 Nice to Have - Optional + quadrant-4 Time Sinks - Avoid + + RPC Tracing: [0.15, 0.92] + TX Submit Trace: [0.3, 0.78] + TX Relay Trace: [0.5, 0.88] + Consensus Trace: [0.72, 0.72] + Peer Msg Trace: [0.85, 0.3] + Ledger Acquire: [0.55, 0.52] +``` + +--- + +## 6.12 Definition of Done + +> **TxQ** = Transaction Queue | **HA** = High Availability + +Clear, measurable criteria for each phase. + +### 6.12.1 Phase 1: Core Infrastructure + +| Criterion | Measurement | Target | +| --------------- | ---------------------------------------------------------- | ---------------------------- | +| SDK Integration | `cmake --build` succeeds with `-DXRPL_ENABLE_TELEMETRY=ON` | ✅ Compiles | +| Runtime Toggle | `enabled=0` produces zero overhead | <0.1% CPU difference | +| Span Creation | Unit test creates and exports span | Span appears in Tempo | +| Configuration | All config options parsed correctly | Config validation tests pass | +| Documentation | Developer guide exists | PR approved | + +**Definition of Done**: All criteria met, PR merged, no regressions in CI. + +### 6.12.2 Phase 2: RPC Tracing + +| Criterion | Measurement | Target | +| ------------------ | ---------------------------------- | -------------------------- | +| Coverage | All RPC commands instrumented | 100% of commands | +| Context Extraction | traceparent header propagates | Integration test passes | +| Attributes | Command, status, duration recorded | Validation script confirms | +| Performance | RPC latency overhead | <1ms p99 | +| Dashboard | Grafana dashboard deployed | Screenshot in docs | + +**Definition of Done**: RPC traces visible in Tempo for all commands, dashboard shows latency distribution. + +### 6.12.3 Phase 3: Transaction Tracing + +| Criterion | Measurement | Target | +| --------------------- | ------------------------------------------------- | -------------------------------------------------------- | +| Local Trace | Submit → validate → TxQ traced | Single-node test passes | +| Cross-Node | Context propagates via protobuf | Multi-node test passes | +| Deterministic TraceID | Same trace_id on all nodes for same tx | Multi-node test: query by txHash[0:16] returns all spans | +| Relay Ordering | Protobuf span_id propagation creates parent-child | Tempo trace tree shows relay chain | +| Graceful Degradation | Old peer drops trace_context | Spans still grouped by deterministic trace_id | +| Relay Visibility | relay_count attribute correct | Spot check 100 txs | +| HashRouter | Deduplication visible in trace | Duplicate txs show suppressed=true | +| Performance | TX throughput overhead | <5% degradation | + +**Definition of Done**: Transaction traces span 3+ nodes in test network with deterministic trace_id correlation, parent-child ordering via protobuf propagation, and performance within bounds. + +### 6.12.4 Phase 4: Consensus Tracing + +| Criterion | Measurement | Target | +| -------------------- | ----------------------------- | ------------------------- | +| Round Tracing | startRound creates root span | Unit test passes | +| Phase Visibility | All phases have child spans | Integration test confirms | +| Proposer Attribution | Proposer ID in attributes | Spot check 50 rounds | +| Timing Accuracy | Phase durations match PerfLog | <5% variance | +| No Consensus Impact | Round timing unchanged | Performance test passes | + +**Definition of Done**: Consensus rounds fully traceable, no impact on consensus timing. + +### 6.12.5 Phase 5: Production Deployment + +| Criterion | Measurement | Target | +| ------------ | ---------------------------- | -------------------------- | +| Collector HA | Multiple collectors deployed | No single point of failure | +| Sampling | Tail sampling configured | 10% base + errors + slow | +| Retention | Data retained per policy | 7 days hot, 30 days warm | +| Alerting | Alerts configured | Error spike, high latency | +| Runbook | Operator documentation | Approved by ops team | +| Training | Team trained | Session completed | + +**Definition of Done**: Telemetry running in production, operators trained, alerts active. + +### 6.12.6 Success Metrics Summary + +| Phase | Primary Metric | Secondary Metric | Deadline | Status | +| -------- | ------------------------------------------------------------------ | --------------------------- | -------------- | ------------------ | +| Phase 1 | SDK compiles and runs | Zero overhead when disabled | End of Week 2 | Active | +| Phase 2 | 100% RPC coverage | <1ms latency overhead | End of Week 4 | Active | +| Phase 3 | Cross-node traces work | <5% throughput impact | End of Week 6 | Active | +| Phase 4 | Consensus fully traced | No consensus timing impact | End of Week 8 | Active | +| Phase 5 | Production deployment | Operators trained | End of Week 9 | Active | +| Phase 6 | StatsD metrics in Prometheus | 3 dashboards operational | End of Week 10 | Active | +| Phase 7 | All metrics via OTLP | No StatsD dependency | End of Week 12 | Active | +| Phase 8 | trace_id in logs + Loki | Tempo↔Loki correlation | End of Week 13 | Active | +| Phase 9 | 68+ new internal metrics in Prom | 2 new dashboards | End of Week 15 | Future Enhancement | +| Phase 10 | Full telemetry stack validated; OTel-sourced regression gate in CI | < 3% CPU overhead proven | End of Week 17 | Future Enhancement | +| Phase 11 | Third-party metrics via receiver | 4 new dashboards + alerting | End of Week 20 | Future Enhancement | + +--- + +## 6.13 Recommended Implementation Order + +Based on ROI analysis, implement in this exact order: + +```mermaid +flowchart TB + subgraph week1["Week 1"] + t1[1. OpenTelemetry SDK
Conan/CMake integration] + t2[2. Telemetry interface
SpanGuard, config] + end + + subgraph week2["Week 2"] + t3[3. RPC ServerHandler
instrumentation] + t4[4. Basic Tempo setup
for testing] + end + + subgraph week3["Week 3"] + t5[5. Transaction submit
tracing] + t6[6. Grafana dashboard
v1] + end + + subgraph week4["Week 4"] + t7[7. Protobuf context
extension] + t8[8. PeerImp tx.relay
instrumentation] + end + + subgraph week5["Week 5"] + t9[9. Multi-node
integration tests] + t10[10. Performance
benchmarks] + end + + subgraph week6_8["Weeks 6-8"] + t11[11. Consensus
instrumentation] + t12[12. Full integration
testing] + end + + subgraph week9["Week 9"] + t13[13. Production
deployment] + t14[14. Documentation
& training] + end + + t1 --> t2 --> t3 --> t4 + t4 --> t5 --> t6 + t6 --> t7 --> t8 + t8 --> t9 --> t10 + t10 --> t11 --> t12 + t12 --> t13 --> t14 + + style week1 fill:#1b5e20,stroke:#0d3d14,color:#fff + style week2 fill:#1b5e20,stroke:#0d3d14,color:#fff + style week3 fill:#bf360c,stroke:#8c2809,color:#fff + style week4 fill:#bf360c,stroke:#8c2809,color:#fff + style week5 fill:#bf360c,stroke:#8c2809,color:#fff + style week6_8 fill:#0d47a1,stroke:#082f6a,color:#fff + style week9 fill:#4a148c,stroke:#2e0d57,color:#fff + style t1 fill:#1b5e20,stroke:#0d3d14,color:#fff + style t2 fill:#1b5e20,stroke:#0d3d14,color:#fff + style t3 fill:#1b5e20,stroke:#0d3d14,color:#fff + style t4 fill:#1b5e20,stroke:#0d3d14,color:#fff + style t5 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style t6 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style t7 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style t8 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style t9 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style t10 fill:#ffe0b2,stroke:#ffcc80,color:#1e293b + style t11 fill:#0d47a1,stroke:#082f6a,color:#fff + style t12 fill:#0d47a1,stroke:#082f6a,color:#fff + style t13 fill:#4a148c,stroke:#2e0d57,color:#fff + style t14 fill:#4a148c,stroke:#2e0d57,color:#fff +``` + +**Reading the diagram:** + +- **Week 1 (tasks 1-2)**: Foundation work -- integrate the OpenTelemetry SDK via Conan/CMake and build the `Telemetry` interface with `SpanGuard` and config parsing. +- **Week 2 (tasks 3-4)**: First observable output -- instrument `ServerHandler` for RPC tracing and stand up Tempo so developers can see traces immediately. +- **Weeks 3-5 (tasks 5-10)**: Transaction lifecycle -- add submit tracing, build the first Grafana dashboard, extend protobuf for cross-node context, instrument `PeerImp` relay, then validate with multi-node integration tests and performance benchmarks. +- **Weeks 6-8 (tasks 11-12)**: Consensus deep-dive -- instrument consensus rounds and phases, then run full integration testing across all instrumented paths. +- **Week 9 (tasks 13-14)**: Go-live -- deploy to production with sampling/alerting configured, and deliver documentation and operator training. +- **Arrow chain (t1 → ... → t14)**: Strict sequential dependency; each task's output is a prerequisite for the next. + +--- + +--- + +## Appendix: External Dashboard Parity + +> Cross-phase plan for reaching parity with the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). Previously a standalone design spec; merged here so the phase plan is self-contained. + +> **Date**: 2026-03-30 +> **Status**: Draft +> **Source**: [realgrapedrop/xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard) +> **Jira Epic**: RIPD-5060 + +### Summary + +Integrate 29 missing metrics, 18 alert rules, and enriched span attributes from the community `xrpl-validator-dashboard` into xrpld's native OpenTelemetry instrumentation. Changes are distributed across phases 2, 3, 4, 6, 7, 9, 10, and 11 of the OTel PR chain. + +### Gap Analysis + +#### Coverage Breakdown (86 external metrics) + +| Status | Count | Notes | +| ----------------- | ----- | -------------------------------------------------------------- | +| Already covered | 30 | peer_count, load_factor, io_latency, uptime, overlay traffic | +| Partially covered | 3 | state_value encoding, NuDB granularity, validation_quorum | +| Missing | 29 | Validation agreement, ledger economy, peer quality, UNL health | +| N/A (external) | 24 | Monitor health, realtime duplicates, system metrics | + +#### Missing Metrics by Category + +| Category | Metrics | Count | +| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | +| Validation Agreement | `validations_sent_total`, `validations_checked_total`, `validation_agreements_total`, `validation_missed_total`, `validation_agreement_pct_1h/24h`, `validation_agreements_1h/24h`, `validation_missed_1h/24h`, `validation_event` | 11 | +| Ledger Economy | `ledgers_closed_total`, `ledger_age_seconds`, `base_fee_xrp`, `reserve_base_xrp`, `reserve_inc_xrp`, `transaction_rate` | 6 | +| State Tracking | `time_in_current_state_seconds`, `state_changes_total`, `validator_state_info` | 3 | +| Peer Quality | `peers_insane`, `peer_latency_p90_ms` | 2 | +| Validator Health | `amendment_blocked`, `unl_expiry_days` | 2 | +| Upgrade Awareness | `peers_higher_version_pct`, `upgrade_recommended` | 2 | +| Storage / Other | `ledger_nudb_bytes`, `jq_trans_overflow_total`, `initial_sync_duration_seconds` | 3 | + +#### Alert Rules (18 total, from external dashboard) + +| Group | Count | Rules | +| ----------- | ----- | ----------------------------------------------------------------------------------------------------------------------- | +| Critical | 8 | Agreement <90%, not proposing, unhealthy state, amendment blocked, UNL expiring, IO latency, load factor, peer count <5 | +| Network | 3 | Peer drop >10%/30%, P90 latency + disconnect correlation | +| Performance | 7 | CPU >80%, memory >90%, disk >85%, job queue overflow, upgrade recommended, tx rate drop, stale ledger | + +--- + +### Branch-to-Change Mapping + +#### Phase 2 — `pratik/otel-phase2-rpc-tracing` + +> **Ref**: Adds to existing Phase 2 task list. Consumed by Phase 7 (MetricsRegistry) and Phase 10 (validation checks). + +**Task 2.8: RPC Span Attribute Enrichment** + +Add node-level health context to every `rpc.command.*` span so operators can correlate RPC behavior with node state. + +New span attributes on `rpc.command.*`: + +| Attribute | Type | Source | Value Example | +| ----------------------------- | ------ | ------------------------------------ | --------------------- | +| `xrpl.node.amendment_blocked` | bool | `app_.getOPs().isAmendmentBlocked()` | `true` | +| `xrpl.node.server_state` | string | `app_.getOPs().strOperatingMode()` | `"full"`, `"syncing"` | + +**File**: `src/xrpld/rpc/detail/RPCHandler.cpp` (in the `rpc.command.*` span creation block, after existing setAttribute calls) + +**Rationale**: RPC is the operator's primary interaction point. When a node is amendment-blocked or degraded, every RPC response is suspect. Tagging spans with this state enables Jaeger queries like `{name=~"rpc.command.*"} | xrpl.node.amendment_blocked = true` to find all RPCs served during a blocked period. + +**Exit Criteria**: + +- [ ] `rpc.command.server_info` spans carry `xrpl.node.amendment_blocked` and `xrpl.node.server_state` attributes +- [ ] No measurable latency impact (attribute values are cached atomics, not computed per-call) + +--- + +#### Phase 3 — `pratik/otel-phase3-tx-tracing` + +> **Ref**: Adds to existing Phase 3 task list. Consumed by Phase 10 (validation checks). + +**Task 3.7: Transaction Span Peer Version Attribute** + +Add the relaying peer's xrpld version to transaction receive spans to enable version-mismatch correlation. + +New span attribute on `tx.receive`: + +| Attribute | Type | Source | Value Example | +| ------------------- | ------ | -------------------- | --------------- | +| `xrpl.peer.version` | string | `peer->getVersion()` | `"xrpld-2.4.0"` | + +**File**: `src/xrpld/overlay/detail/PeerImp.cpp` (in the `tx.receive` span block, after existing `xrpl.peer.id` setAttribute) + +**Rationale**: Transaction relay is where version mismatches cause subtle serialization or validation bugs. Tracing "this tx came from a v2.3.0 peer" helps diagnose compatibility issues during network upgrades. + +**Exit Criteria**: + +- [ ] `tx.receive` spans carry `xrpl.peer.version` attribute with a non-empty version string +- [ ] Attribute is omitted (not empty-string) when `getVersion()` returns empty + +--- + +#### Phase 4 — `pratik/otel-phase4-consensus-tracing` + +> **Ref**: Adds to existing Phase 4 task list. Provides the span-level foundation that Phase 7 (ValidationTracker) builds upon. Consumed by Phase 10 (validation checks). + +**Task 4.8: Consensus Validation Span Enrichment** + +Add ledger hash and validation type to validation spans on both send and receive paths. This enables trace-level agreement analysis — filter by ledger hash to see which validators agreed. + +New span attributes on `consensus.validation.send`: + +| Attribute | Type | Source | Value Example | +| ----------------------------- | ------ | --------------------------------------- | --------------------------- | +| `xrpl.validation.ledger_hash` | string | Ledger hash from `validate()` call args | `"A1B2C3..."` (64-char hex) | +| `xrpl.validation.full` | bool | Whether this is a full validation | `true` | + +New span attributes on `peer.validation.receive`: + +| Attribute | Type | Source | Value Example | +| ---------------------------------- | ------ | ------------------------------------- | --------------------------- | +| `xrpl.peer.validation.ledger_hash` | string | From deserialized STValidation object | `"A1B2C3..."` (64-char hex) | +| `xrpl.peer.validation.full` | bool | From STValidation flags | `true` | + +New span attributes on `consensus.accept`: + +| Attribute | Type | Source | Value Example | +| ------------------------------------ | ----- | ---------------------------------------- | ------------- | +| `xrpl.consensus.validation_quorum` | int64 | `app_.validators().quorum()` | `28` | +| `xrpl.consensus.proposers_validated` | int64 | `result.proposers` from consensus result | `35` | + +**Files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` (validation.send and accept spans) +- `src/xrpld/overlay/detail/PeerImp.cpp` (peer.validation.receive span) + +**Rationale**: The external dashboard's most valuable feature is validation agreement tracking. By recording the ledger hash on both outgoing and incoming validation spans, we create the raw data for agreement analysis at the trace level. Phase 7's ValidationTracker builds the metric-level aggregation on top of this. + +**Exit Criteria**: + +- [ ] `consensus.validation.send` spans carry `xrpl.validation.ledger_hash` and `xrpl.validation.full` +- [ ] `peer.validation.receive` spans carry `xrpl.peer.validation.ledger_hash` and `xrpl.peer.validation.full` +- [ ] `consensus.accept` spans carry `xrpl.consensus.validation_quorum` and `xrpl.consensus.proposers_validated` +- [ ] Ledger hash attributes match between send and receive for the same ledger + +--- + +#### Phase 6 — `pratik/otel-phase6-statsd` + +> **Ref**: Adds to existing Phase 6 scope. No separate task list file exists for Phase 6 per project convention. + +**Addition: Bridge `peerDisconnectsCharges_` metric** + +The overlay already tracks resource-limit disconnects via `OverlayImpl::Stats::peerDisconnectsCharges_` (a `beast::insight::Gauge`). This metric is registered but not included in the StatsD bridge mapping. + +**What to do**: + +- Ensure `overlay_peer_disconnects_charges` appears in the StatsD-to-Prometheus metric name mapping +- Verify the metric appears in Prometheus after StatsD bridge is active + +**File**: `src/xrpld/overlay/detail/OverlayImpl.cpp` + +**Prometheus name**: `overlay_peer_disconnects_charges` + +--- + +#### Phase 7 — `pratik/otel-phase7-native-metrics` + +> **Ref**: Adds to existing Phase 7 task list. This is the largest addition. Depends on Phase 4 span attributes for validation tracking context. Consumed by Phase 9 (dashboards), Phase 10 (validation), Phase 11 (alerts). + +**Task 7.8: ValidationTracker — Validation Agreement Computation** + +The most valuable missing component. A stateful class that tracks whether our validator's validations agree with network consensus, maintaining rolling 1h and 24h windows. + +**Architecture**: + +``` + + consensus.validation.send ─────> ValidationTracker ──────> MetricsRegistry + (records our validation (reconciles after (exports agreement + for ledger X) 8s grace period) gauges every 10s) + + ledger.validate ───────────────> ValidationTracker + (records which ledger (marks ledger X as + network validated) agreed or missed) +``` + +**Design**: + +```cpp +/// Tracks validation agreement between this node and network consensus. +/// +/// ValidationTracker +/// ├── recordOurValidation(ledgerHash, ledgerSeq) // called when we send +/// ├── recordNetworkValidation(ledgerHash, seq) // called on ledger validate +/// ├── reconcile() // called periodically (timer) +/// ├── agreementPct1h() -> double // 0.0-100.0 +/// ├── agreementPct24h() -> double +/// ├── agreements1h() -> uint64_t +/// ├── missed1h() -> uint64_t +/// ├── agreements24h() -> uint64_t +/// ├── missed24h() -> uint64_t +/// ├── totalAgreements() -> uint64_t +/// ├── totalMissed() -> uint64_t +/// ├── totalValidationsSent() -> uint64_t +/// └── totalValidationsChecked() -> uint64_t // all network validations seen +class ValidationTracker +{ + // Ring buffer of pending ledger events (max 1000) + struct LedgerEvent { + uint256 ledgerHash; + LedgerIndex seq; + TimePoint closeTime; + bool weValidated = false; // did we send a validation for this ledger? + bool networkValidated = false; // did network validate this ledger? + bool reconciled = false; // has 8s grace period elapsed? + bool agreed = false; // after reconciliation: did we agree? + }; + + // Sliding window deques for pre-computed window stats + struct WindowEvent { + TimePoint time; + bool agreed; + }; + std::deque window1h_; // events in last 1 hour + std::deque window24h_; // events in last 24 hours + + // Reconciliation: 8s grace period after ledger close. + // If our validation hasn't arrived by then, mark as missed. + // 5-minute late repair: if a late validation arrives, correct the miss. + static constexpr auto kGracePeriod = std::chrono::seconds(8); + static constexpr auto kLateRepairWindow = std::chrono::minutes(5); +}; +``` + +**Recording sites** (modifications to consensus code from Phase 7 branch): + +| Hook Point | File | What to Record | +| ---------------------------- | ------------------- | ----------------------------------------------------------------------- | +| `validate()` in `doAccept()` | RCLConsensus.cpp | `tracker.recordOurValidation(ledgerHash, seq)` | +| `onValidation()` callback | RCLValidations path | `tracker.recordNetworkValidation(...)` — increment `validationsChecked` | +| LedgerMaster fully-validated | LedgerMaster.cpp | `tracker.recordNetworkValidation(validatedHash, seq)` | + +**Key new files**: + +- `src/xrpld/telemetry/ValidationTracker.h` +- `src/xrpld/telemetry/detail/ValidationTracker.cpp` + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.h` (add ValidationTracker member) +- `src/xrpld/telemetry/MetricsRegistry.cpp` (add gauge callback reading from tracker) +- `src/xrpld/app/consensus/RCLConsensus.cpp` (add recording hooks) +- `src/xrpld/app/ledger/detail/LedgerMaster.cpp` (add recording hook) + +**Exit Criteria**: + +- [ ] `ValidationTracker` correctly tracks agreement with 8s grace period +- [ ] 5-minute late repair corrects false-positive misses +- [ ] Thread-safe (atomics + mutex for window deques) +- [ ] Rolling windows correctly evict stale entries +- [ ] Unit tests for: normal agreement, missed validation, late repair, window eviction + +--- + +**Task 7.9: Validator Health Observable Gauges** + +New MetricsRegistry observable gauge for amendment, UNL, and quorum health. + +| Gauge Name | Label `metric=` | Type | Source | +| ------------------ | ------------------- | ------ | ------------------------------------------------- | +| `validator_health` | `amendment_blocked` | int64 | `app_.getOPs().isAmendmentBlocked()` → 0/1 | +| | `unl_blocked` | int64 | `app_.getOPs().isUNLBlocked()` → 0/1 | +| | `unl_expiry_days` | double | `app_.validators().expires()` → days until expiry | +| | `validation_quorum` | int64 | `app_.validators().quorum()` | + +**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` (new gauge callback in `registerAsyncGauges()`) + +**Exit Criteria**: + +- [ ] All 4 label values emitted every 10s +- [ ] `unl_expiry_days` is negative when expired, positive when active +- [ ] Values visible in Prometheus + +--- + +**Task 7.10: Peer Quality Observable Gauges** + +New MetricsRegistry observable gauge for peer health aggregates. + +| Gauge Name | Label `metric=` | Type | Source | +| -------------- | -------------------------- | ------ | ------------------------------------------ | +| `peer_quality` | `peer_latency_p90_ms` | double | Iterate peers, compute P90 from `latency_` | +| | `peers_insane_count` | int64 | Count peers with `tracking_ == diverged` | +| | `peers_higher_version_pct` | double | Compare `getVersion()` to own version | +| | `upgrade_recommended` | int64 | 1 if `peers_higher_version_pct > 60%` | + +**Implementation note**: The callback iterates `app_.overlay().foreach(...)` to collect per-peer latency and version data. This runs every 10s on the metrics reader thread — acceptable overhead for ~50-200 peers. + +**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Exit Criteria**: + +- [ ] P90 latency computed correctly (sort peer latencies, pick 90th percentile) +- [ ] Insane count matches `peers` RPC output +- [ ] Version comparison handles format variations (e.g., "xrpld-2.4.0-rc1") +- [ ] Values visible in Prometheus + +--- + +**Task 7.11: Ledger Economy Observable Gauges** + +New MetricsRegistry observable gauge for fee and ledger metrics. + +| Gauge Name | Label `metric=` | Type | Source | +| ---------------- | -------------------- | ------ | ----------------------------------------- | +| `ledger_economy` | `base_fee_xrp` | double | `app_.getFeeTrack().getBaseFee()` → drops | +| | `reserve_base_xrp` | double | From validated ledger fee settings | +| | `reserve_inc_xrp` | double | From validated ledger fee settings | +| | `ledger_age_seconds` | double | `now - lastValidatedCloseTime` | +| | `transaction_rate` | double | Derived: tx count delta / time delta | + +**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Exit Criteria**: + +- [ ] Fee values match `server_info` RPC output +- [ ] `ledger_age_seconds` increases monotonically between ledger closes, resets on close +- [ ] `transaction_rate` is smoothed (rolling average, not instantaneous) + +--- + +**Task 7.12: State Tracking Observable Gauges** + +New MetricsRegistry observable gauge for node state duration. + +| Gauge Name | Label `metric=` | Type | Source | +| ---------------- | ------------------------------- | ------ | ------------------------------------------------ | +| `state_tracking` | `state_value` | int64 | 0-7 numeric encoding matching external dashboard | +| | `time_in_current_state_seconds` | double | `now - lastModeChangeTime` | + +**State value encoding**: + +xrpld's `OperatingMode` enum maps 0-4 (DISCONNECTED through FULL). The external dashboard extends this to 0-6 by combining operating mode with consensus participation: + +| Value | State | Source | +| ----- | ------------ | ----------------------------------------------------------- | +| 0 | disconnected | `OperatingMode::DISCONNECTED` | +| 1 | connected | `OperatingMode::CONNECTED` | +| 2 | syncing | `OperatingMode::SYNCING` | +| 3 | tracking | `OperatingMode::TRACKING` | +| 4 | full | `OperatingMode::FULL` and not validating | +| 5 | validating | `OperatingMode::FULL` and `mConsensus.validating()` is true | +| 6 | proposing | `OperatingMode::FULL` and consensus mode is `proposing` | + +**Note**: Values 5-6 require checking both `OperatingMode` and `ConsensusMode`. The callback should derive these from `app_.getOPs().getOperatingMode()` combined with `mConsensus.mode()`. If operating mode is FULL and consensus is proposing → 6; if FULL and validating → 5; otherwise use the raw OperatingMode enum value. + +**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Exit Criteria**: + +- [ ] `state_value` matches external dashboard encoding +- [ ] `time_in_current_state_seconds` resets on mode change + +--- + +**Task 7.13: Storage Detail Observable Gauge** + +| Gauge Name | Label `metric=` | Type | Source | +| ---------------- | --------------- | ----- | ---------------------------------------- | +| `storage_detail` | `nudb_bytes` | int64 | NuDB backend file size (filesystem stat) | + +**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Exit Criteria**: + +- [ ] NuDB file size reported in bytes +- [ ] Gracefully returns 0 if NuDB not configured + +--- + +**Task 7.14: New Synchronous Counters** + +New counters incremented at event sites. Declared in MetricsRegistry, recording sites added in consensus/overlay/network code. + +| Counter Name | Increment Site | Source File | +| ----------------------------- | -------------------------------- | --------------------- | +| `ledgers_closed_total` | `onAccept()` in consensus | RCLConsensus.cpp | +| `validations_sent_total` | `validate()` in consensus | RCLConsensus.cpp | +| `validations_checked_total` | Network validation received | LedgerMaster.cpp | +| `validation_agreements_total` | ValidationTracker reconciliation | ValidationTracker.cpp | +| `validation_missed_total` | ValidationTracker reconciliation | ValidationTracker.cpp | +| `state_changes_total` | `setMode()` in NetworkOPs | NetworkOPs.cpp | +| `jq_trans_overflow_total` | Job queue overflow path | JobQueue.cpp | + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (counter declarations) +- `src/xrpld/app/consensus/RCLConsensus.cpp` (recording: ledgers_closed, validations_sent) +- `src/xrpld/app/ledger/detail/LedgerMaster.cpp` (recording: validations_checked) +- `src/xrpld/app/misc/NetworkOPs.cpp` (recording: state_changes) + +**Exit Criteria**: + +- [ ] All 7 counters monotonically increase during normal operation +- [ ] Counter values match expected rates (e.g., ledgers_closed ≈ 1 per 3-5s) +- [ ] Values visible in Prometheus + +--- + +**Task 7.15: Validation Agreement Observable Gauge** + +Reads from the `ValidationTracker` (Task 7.8) to export rolling window stats. + +| Gauge Name | Label `metric=` | Type | Source | +| ---------------------- | ------------------- | ------ | --------------------------- | +| `validation_agreement` | `agreement_pct_1h` | double | `tracker.agreementPct1h()` | +| | `agreements_1h` | int64 | `tracker.agreements1h()` | +| | `missed_1h` | int64 | `tracker.missed1h()` | +| | `agreement_pct_24h` | double | `tracker.agreementPct24h()` | +| | `agreements_24h` | int64 | `tracker.agreements24h()` | +| | `missed_24h` | int64 | `tracker.missed24h()` | + +**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Exit Criteria**: + +- [ ] Agreement percentages in range [0.0, 100.0] +- [ ] Window stats match manual count from validation counters +- [ ] Percentages stabilize after 1h/24h of operation + +--- + +#### Phase 9 — `pratik/otel-phase9-metric-gap-fill` + +> **Ref**: Adds to existing Phase 9 task list. Depends on Phase 7 gauges/counters. Consumed by Phase 10 (dashboard load checks). + +**Task 9.11: Validator Health Dashboard** + +New Grafana dashboard: `validator-health.json` + +| Panel | Type | PromQL | +| -------------------------- | ---------- | -------------------------------------------------------- | +| Agreement % (1h) | stat | `validation_agreement{metric="agreement_pct_1h"}` | +| Agreement % (24h) | stat | `validation_agreement{metric="agreement_pct_24h"}` | +| Agreements vs Missed (1h) | bargauge | `agreements_1h` and `missed_1h` side by side | +| Agreements vs Missed (24h) | bargauge | `agreements_24h` and `missed_24h` side by side | +| Validation Rate | stat | `rate(validations_sent_total[5m]) * 60` | +| Validations Checked Rate | stat | `rate(validations_checked_total[5m]) * 60` | +| Amendment Blocked | stat | `validator_health{metric="amendment_blocked"}` | +| UNL Expiry (days) | stat | `validator_health{metric="unl_expiry_days"}` | +| Validation Quorum | stat | `validator_health{metric="validation_quorum"}` | +| State Value Timeline | timeseries | `state_tracking{metric="state_value"}` | +| Time in Current State | stat | `state_tracking{metric="time_in_current_state_seconds"}` | +| State Changes Rate | stat | `rate(state_changes_total[1h])` | +| Ledgers Closed Rate | stat | `rate(ledgers_closed_total[5m]) * 60` | + +**Dashboard conventions**: `$node` template variable for `service_instance_id` filtering, dark theme, matching existing panel sizes and color schemes. + +--- + +**Task 9.12: Peer Quality Dashboard** + +New Grafana dashboard: `peer-quality.json` + +| Panel | Type | PromQL | +| ---------------------- | ---------- | -------------------------------------------------------- | +| P90 Peer Latency | timeseries | `peer_quality{metric="peer_latency_p90_ms"}` | +| Insane/Diverged Peers | stat | `peer_quality{metric="peers_insane_count"}` | +| Higher Version Peers % | stat | `peer_quality{metric="peers_higher_version_pct"}` | +| Upgrade Recommended | stat | `peer_quality{metric="upgrade_recommended"}` | +| Resource Disconnects | timeseries | `overlay_peer_disconnects_charges` | +| Inbound vs Outbound | bargauge | `peer_finder_active_inbound_peers`, `..._outbound_peers` | + +--- + +**Task 9.13: Ledger Economy Dashboard Panels** + +Add a "Ledger Economy" row to the existing `node-health.json` dashboard: + +| Panel | Type | PromQL | +| -------------------- | ---------- | --------------------------------------------- | +| Base Fee (drops) | stat | `ledger_economy{metric="base_fee_xrp"}` | +| Reserve Base (drops) | stat | `ledger_economy{metric="reserve_base_xrp"}` | +| Reserve Inc (drops) | stat | `ledger_economy{metric="reserve_inc_xrp"}` | +| Ledger Age | stat | `ledger_economy{metric="ledger_age_seconds"}` | +| Transaction Rate | timeseries | `ledger_economy{metric="transaction_rate"}` | + +--- + +#### Phase 10 — `pratik/otel-phase10-workload-validation` + +> **Ref**: Adds to existing Phase 10 task list. Validates all additions from Phases 2-9. + +**Task 10.6: External Dashboard Parity Validation Checks** + +Add checks to `validate_telemetry.py` for all new span attributes and metrics. + +**New span attribute checks (~8)**: + +| Span Name | New Attribute | +| --------------------------- | ------------------------------------ | +| `rpc.command.server_info` | `xrpl.node.amendment_blocked` | +| `rpc.command.server_info` | `xrpl.node.server_state` | +| `tx.receive` | `xrpl.peer.version` | +| `consensus.validation.send` | `xrpl.validation.ledger_hash` | +| `consensus.validation.send` | `xrpl.validation.full` | +| `peer.validation.receive` | `xrpl.peer.validation.ledger_hash` | +| `consensus.accept` | `xrpl.consensus.validation_quorum` | +| `consensus.accept` | `xrpl.consensus.proposers_validated` | + +**New metric existence checks (~13)**: + +| Metric Name | +| -------------------------------------------------- | +| `validation_agreement{metric="agreement_pct_1h"}` | +| `validation_agreement{metric="agreement_pct_24h"}` | +| `validator_health{metric="amendment_blocked"}` | +| `validator_health{metric="unl_expiry_days"}` | +| `peer_quality{metric="peer_latency_p90_ms"}` | +| `peer_quality{metric="peers_insane_count"}` | +| `ledger_economy{metric="base_fee_xrp"}` | +| `ledger_economy{metric="transaction_rate"}` | +| `state_tracking{metric="state_value"}` | +| `ledgers_closed_total` | +| `validations_sent_total` | +| `state_changes_total` | +| `storage_detail{metric="nudb_bytes"}` | + +**New dashboard load checks (~3)**: + +| Dashboard | +| ----------------------- | +| `validator-health` | +| `peer-quality` | +| `node-health` (updated) | + +**New metric value sanity checks (~4)**: + +| Check | Condition | +| ----------------------------- | ----------------- | +| `validation_agreement_pct_1h` | in [0, 100] | +| `unl_expiry_days` | > 0 (not expired) | +| `peer_latency_p90_ms` | > 0 (peers exist) | +| `state_value` | in [0, 7] | + +**Total new checks: ~28** (bringing total from 73 to ~101) + +--- + +#### Phase 11 — (future branch) + +> **Ref**: Adds to existing Phase 11 task list. Depends on Phase 7 metrics and Phase 9 dashboards. + +**Task 11.9: Alert Rules from External Dashboard** + +Port 18 alert rules from the external `xrpl-validator-dashboard` to Grafana alerting provisioning. + +**Critical Group** (8 rules, eval interval 10s): + +| Rule | Condition | For | +| ------------------- | ------------------------------------------------------- | --- | +| Agreement Below 90% | `validation_agreement{metric="agreement_pct_24h"} < 90` | 30s | +| Not Proposing | `state_tracking{metric="state_value"} < 6` | 10s | +| Unhealthy State | `state_tracking{metric="state_value"} < 4` | 10s | +| Amendment Blocked | `validator_health{metric="amendment_blocked"} == 1` | 1m | +| UNL Expiring | `validator_health{metric="unl_expiry_days"} < 14` | 1h | +| High IO Latency | `histogram_quantile(0.95, ios_latency_bucket) > 50` | 1m | +| High Load Factor | `load_factor_metrics{metric="load_factor"} > 1000` | 1m | +| Peer Count Critical | `server_info{metric="peers"} < 5` | 1m | + +**Network Group** (3 rules, eval interval 10s): + +| Rule | Condition | For | +| ------------------------- | ----------------------------------------------------------- | --- | +| Peer Drop >10% | `delta(server_info{metric="peers"}[30s]) / ... * 100 < -10` | 30s | +| Peer Drop >30% | Same formula, threshold -30 | 30s | +| P90 Latency + Disconnects | `peer_latency_p90_ms > 500 AND rate(disconnects) > 0` | 2m | + +**Performance Group** (7 rules, eval interval 10s): + +| Rule | Condition | For | +| ------------------- | ------------------------------------------------------ | --- | +| CPU High | Per-core CPU > 80% | 2m | +| Memory Critical | Memory usage > 90% | 1m | +| Disk Warning | Disk usage > 85% | 2m | +| Job Queue Overflow | `rate(jq_trans_overflow_total[5m]) > 0` | 1m | +| Upgrade Recommended | `peer_quality{metric="peers_higher_version_pct"} > 60` | 1m | +| TX Rate Drop | Transaction rate dropped > 50% in 5m window | 5m | +| Stale Ledger | `ledger_economy{metric="ledger_age_seconds"} > 30` | 1m | + +**Notification channels**: Template configs for Email/SMTP, Discord, Slack, PagerDuty. + +**Files**: + +- `docker/telemetry/grafana/alerting/alert-rules.yaml` (new or extend existing) +- `docker/telemetry/grafana/alerting/contact-points.yaml` +- `docker/telemetry/grafana/alerting/notification-policies.yaml` + +--- + +**Task 11.10: Dual-Datasource Architecture Documentation** + +Document the external dashboard's "fast path" pattern as a future optimization for real-time panels: + +- **Pattern**: A lightweight Prometheus scrape endpoint (separate from OTLP pipeline) that polls critical metrics every 2-5s, bypassing the 10s OTLP metric reader interval and Prometheus scrape interval. +- **Use case**: Real-time state panels (server state, ledger age, peer count) where 10-15s latency is too slow. +- **Decision**: Document as a future option, not implement now. Current 10s interval is acceptable for v1. + +**File**: `OpenTelemetryPlan/Phase11_taskList.md` (documentation task, no code) + +--- + +### Documentation Updates + +#### `docs/telemetry-runbook.md` (on Phase 9 branch) + +Add new sections after "Phase 9: OTel Metrics Alerting Rules": + +1. **Validator Health Monitoring** — explains agreement tracking, amendment blocked, UNL expiry, with example PromQL queries +2. **Peer Quality Monitoring** — explains P90 latency, insane peers, version awareness +3. **Ledger Economy Monitoring** — explains fee/reserve gauges, transaction rate, ledger age +4. **Validation Agreement Explained** — operator-facing explanation of the reconciliation algorithm (8s grace, 5m late repair), what "missed" means, and when to worry + +#### `OpenTelemetryPlan/09-data-collection-reference.md` (on Phase 9 branch) + +Add new metric tables in a "Phase 7+: External Dashboard Parity" section covering all 29 new metrics with their gauge names, label values, types, and sources. + +--- + +### Cross-Phase Dependency Chain + +``` +Phase 2 (span attrs: amendment_blocked, server_state) +Phase 3 (span attrs: peer.version) +Phase 4 (span attrs: validation.ledger_hash, validation.full, quorum) +Phase 6 (StatsD bridge: peerDisconnectsCharges) + │ + ├── all above rebase into ──> + │ +Phase 7 (ValidationTracker + 7 gauges + 7 counters + agreement gauge) + │ +Phase 9 (3 dashboards + ledger economy panels + runbook + data-collection-ref) + │ +Phase 10 (28 new validation checks in validate_telemetry.py) + │ +Phase 11 (18 alert rules + dual-datasource docs) +``` + +### Rebase Strategy + +After committing changes to each branch (starting from Phase 2): + +1. Commit on `pratik/otel-phase2-rpc-tracing` +2. Rebase `phase3` onto `phase2`, resolve conflicts (task list files only — low risk) +3. Commit on `phase3`, rebase `phase4` onto `phase3` +4. Continue through chain: 4 → 5 → 5b → 6 → 7 → 8 → 9 → 10 +5. Force-push-with-lease all affected branches + +Since these are documentation-only changes (task list .md files), merge conflicts should be minimal — each file is unique to its branch. + +_Previous: [Configuration Reference](./05-configuration-reference.md)_ | _Next: [Observability Backends](./07-observability-backends.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/07-observability-backends.md b/OpenTelemetryPlan/07-observability-backends.md new file mode 100644 index 00000000000..4ebb6028fd2 --- /dev/null +++ b/OpenTelemetryPlan/07-observability-backends.md @@ -0,0 +1,407 @@ +# Observability Backend Recommendations + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Implementation Phases](./06-implementation-phases.md) | [Appendix](./08-appendix.md) + +--- + +## 7.1 Development/Testing Backends + +> **OTLP** = OpenTelemetry Protocol + +| Backend | Pros | Cons | Use Case | +| ---------- | ----------------------------------- | ---------------------- | ------------------- | +| **Tempo** | Cost-effective, Grafana integration | Requires Grafana stack | Local dev, CI, Prod | +| **Zipkin** | Simple, lightweight | Basic features | Quick prototyping | + +### Quick Start with Tempo + +```bash +# Start Tempo with OTLP support +docker run -d --name tempo \ + -p 3200:3200 \ + -p 4317:4317 \ + -p 4318:4318 \ + grafana/tempo:2.6.1 +``` + +--- + +## 7.2 Production Backends + +> **APM** = Application Performance Monitoring + +| Backend | Pros | Cons | Use Case | +| ----------------- | ----------------------------------------- | ---------------------- | --------------------------- | +| **Grafana Tempo** | Cost-effective, Grafana integration | Requires Grafana stack | Most production deployments | +| **Elastic APM** | Full observability stack, log correlation | Resource intensive | Existing Elastic users | +| **Honeycomb** | Excellent query, high cardinality | SaaS cost | Deep debugging needs | +| **Datadog APM** | Full platform, easy setup | SaaS cost | Enterprise with budget | + +### Backend Selection Flowchart + +```mermaid +flowchart TD + start[Select Backend] --> budget{Budget
Constraints?} + + budget -->|Yes| oss[Open Source] + budget -->|No| saas{Prefer
SaaS?} + + oss --> existing{Existing
Stack?} + existing -->|Grafana| tempo[Grafana Tempo] + existing -->|Elastic| elastic[Elastic APM] + existing -->|None| tempo + + saas -->|Yes| enterprise{Enterprise
Support?} + saas -->|No| oss + + enterprise -->|Yes| datadog[Datadog APM] + enterprise -->|No| honeycomb[Honeycomb] + + tempo --> final[Configure Collector] + elastic --> final + honeycomb --> final + datadog --> final + + style start fill:#0f172a,stroke:#020617,color:#fff + style budget fill:#334155,stroke:#1e293b,color:#fff + style oss fill:#1e293b,stroke:#0f172a,color:#fff + style existing fill:#334155,stroke:#1e293b,color:#fff + style saas fill:#334155,stroke:#1e293b,color:#fff + style enterprise fill:#334155,stroke:#1e293b,color:#fff + style final fill:#0f172a,stroke:#020617,color:#fff + style tempo fill:#1b5e20,stroke:#0d3d14,color:#fff + style elastic fill:#bf360c,stroke:#8c2809,color:#fff + style honeycomb fill:#0d47a1,stroke:#082f6a,color:#fff + style datadog fill:#4a148c,stroke:#2e0d57,color:#fff +``` + +**Reading the diagram:** + +- **Budget Constraints? (Yes)**: Leads to open-source options. If you already run Grafana or Elastic, pick the matching backend; otherwise default to Grafana Tempo. +- **Budget Constraints? (No) → Prefer SaaS?**: If you want a managed service, choose between Datadog (enterprise support) and Honeycomb (developer-focused). If not, fall back to open-source. +- **Terminal nodes (Tempo / Elastic / Honeycomb / Datadog)**: Each represents a concrete backend choice, all of which feed into the same final step. +- **Configure Collector**: Regardless of backend, you always finish by configuring the OTel Collector to export to your chosen destination. + +--- + +## 7.3 Recommended Production Architecture + +> **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring | **HA** = High Availability + +```mermaid +flowchart TB + subgraph validators["Validator Nodes"] + v1[xrpld
Validator 1] + v2[xrpld
Validator 2] + end + + subgraph stock["Stock Nodes"] + s1[xrpld
Stock 1] + s2[xrpld
Stock 2] + end + + subgraph collector["OTel Collector Cluster"] + c1[Collector
DC1] + c2[Collector
DC2] + end + + subgraph backends["Storage Backends"] + tempo[(Grafana
Tempo)] + elastic[(Elastic
APM)] + archive[(S3/GCS
Archive)] + end + + subgraph ui["Visualization"] + grafana[Grafana
Dashboards] + end + + v1 -->|OTLP| c1 + v2 -->|OTLP| c1 + s1 -->|OTLP| c2 + s2 -->|OTLP| c2 + + c1 --> tempo + c1 --> elastic + c2 --> tempo + c2 --> archive + + tempo --> grafana + elastic --> grafana + + %% Note: simplified single-collector-per-DC topology shown for clarity + + style validators fill:#b71c1c,stroke:#7f1d1d,color:#ffffff + style stock fill:#0d47a1,stroke:#082f6a,color:#ffffff + style collector fill:#bf360c,stroke:#8c2809,color:#ffffff + style backends fill:#1b5e20,stroke:#0d3d14,color:#ffffff + style ui fill:#4a148c,stroke:#2e0d57,color:#ffffff +``` + +**Reading the diagram:** + +- **Validator / Stock Nodes**: All xrpld nodes emit trace data via OTLP. Validators and stock nodes are grouped separately because they may reside in different network zones. +- **Collector Cluster (DC1, DC2)**: Regional collectors receive OTLP from nodes in their datacenter, apply processing (sampling, enrichment), and fan out to multiple backends. Enrichment includes deployment-tier tagging: each collector stamps `deployment.environment` and (as a fallback) `xrpl.network.type` so one Grafana stack can filter data from many collectors by tier. +- **Storage Backends**: Tempo and Elastic provide queryable trace storage; S3/GCS Archive provides long-term cold storage for compliance or post-incident analysis. +- **Grafana Dashboards**: The single visualization layer that queries both Tempo and Elastic, giving operators a unified view of all traces. +- **Data flow direction**: Nodes → Collectors → Storage → Grafana. Each arrow represents a network hop; minimizing collector-to-backend hops reduces latency. + +> **Note**: Production deployments should use multiple collector instances behind a load balancer for high availability. The diagram shows a simplified single-collector topology for clarity. + +--- + +## 7.4 Architecture Considerations + +### 7.4.1 Collector Placement + +| Strategy | Description | Pros | Cons | +| ------------- | -------------------- | ------------------------ | ----------------------- | +| **Sidecar** | Collector per node | Isolation, simple config | Resource overhead | +| **DaemonSet** | Collector per host | Shared resources | Complexity | +| **Gateway** | Central collector(s) | Centralized processing | Single point of failure | + +**Recommendation**: Use **Gateway** pattern with regional collectors for xrpld networks: + +- One collector cluster per datacenter/region +- Tail-based sampling at collector level +- Multiple export destinations for redundancy + +### 7.4.2 Sampling Strategy + +```mermaid +flowchart LR + subgraph head["Head Sampling (Node)"] + hs[Node-level head sampling
fixed at 100%
not configurable] + end + + subgraph tail["Tail Sampling (Collector)"] + ts1[Keep all errors] + ts2[Keep slow >5s] + ts3[Keep 10% rest] + end + + head --> tail + + ts1 --> final[Final Traces] + ts2 --> final + ts3 --> final + + style head fill:#0d47a1,stroke:#082f6a,color:#fff + style tail fill:#1b5e20,stroke:#0d3d14,color:#fff + style hs fill:#0d47a1,stroke:#082f6a,color:#fff + style ts1 fill:#1b5e20,stroke:#0d3d14,color:#fff + style ts2 fill:#1b5e20,stroke:#0d3d14,color:#fff + style ts3 fill:#1b5e20,stroke:#0d3d14,color:#fff + style final fill:#bf360c,stroke:#8c2809,color:#fff +``` + +**Reading the diagram:** + +- **Head Sampling (Node)**: xrpld pins head sampling at 100% (sample everything) and does not expose a configurable ratio. This is intentional: a per-node ratio would let different nodes make divergent keep/drop decisions for the same distributed trace, producing broken/partial traces. xrpld uses a `ParentBased` sampler so spans inheriting a remote parent honor the upstream decision. Volume reduction is delegated to the collector's tail sampling. +- **Tail Sampling (Collector)**: The second filter -- the collector inspects completed traces and applies rules: keep all errors, keep anything slower than 5 seconds, and keep 10% of the remainder. +- **Arrow head → tail**: All head-sampled traces flow to the collector, where tail sampling further reduces volume while preserving the most valuable data. +- **Final Traces**: The output after both sampling stages; this is what gets stored and queried. The two-stage approach balances cost with debuggability. + +### 7.4.3 Data Retention + +| Environment | Hot Storage | Warm Storage | Cold Archive | +| ----------- | ----------- | ------------ | ------------ | +| Development | 24 hours | N/A | N/A | +| Staging | 7 days | N/A | N/A | +| Production | 7 days | 30 days | many years | + +--- + +## 7.5 Integration Checklist + +- [ ] Choose primary backend (Tempo recommended for cost/features) +- [ ] Deploy collector cluster with high availability +- [ ] Configure tail-based sampling for error/latency traces +- [ ] Set up Grafana dashboards for trace visualization +- [ ] Configure alerts for trace anomalies +- [ ] Establish data retention policies +- [ ] Test trace correlation with logs and metrics + +--- + +## 7.6 Grafana Dashboard Examples + +Pre-built dashboards for xrpld observability. + +### 7.6.1 Consensus Health Dashboard + +A Tempo-backed dashboard (uid `xrpld-consensus-health`) with four panels, all driven by TraceQL: + +- **Consensus Round Duration** (timeseries, ms): average `consensus.round` span duration per node instance, with yellow/red thresholds at 4s/5s. +- **Phase Duration Breakdown** (barchart): average duration of `consensus.phase.*` spans grouped by span name. +- **Proposers per Round** (stat): average of the `span.proposers` attribute on `consensus.round` spans. +- **Recent Slow Rounds (>5s)** (table): `consensus.round` spans filtered to `duration > 5s`. + +Each panel's TraceQL query is described inline in its bullet above. + +### 7.6.2 Node Overview Dashboard + +A Tempo-backed dashboard (uid `xrpld-node-overview`) with four panels: + +- **Active Nodes** (stat): count of distinct `resource.service.instance.id` values seen for the `xrpld` service. +- **Total Transactions (1h)** (stat): count of `tx.receive` spans. +- **Error Rate** (gauge, percent): ratio of `status.code=error` spans to all spans, with yellow/red thresholds at 1%/5%. +- **Service Map** (nodeGraph): Tempo-generated service dependency graph. + +### 7.6.3 Alert Rules + +Grafana provisions three TraceQL-based alert rules (group `xrpld-tracing-alerts`, evaluated every 1m) against the Tempo datasource: + +- **Consensus Round Slow** (warning, `for: 5m`): fires when average `consensus.round` duration exceeds 5s. + + ``` + {resource.service.name="xrpld" && name="consensus.round"} | avg(duration) > 5s + ``` + +- **RPC Error Rate Spike** (critical, `for: 2m`): fires when the error rate across `rpc.command.*` spans exceeds 5%. Error _rate_ is a ratio, so it must divide the error-span rate by the total-span rate — a single TraceQL `rate()` returns spans/second, not a percentage, and would fire on traffic volume alone. This uses span metrics emitted by the collector's `spanmetrics` connector (Prometheus datasource), not a TraceQL query: + + ``` + sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m])) + / + sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m])) + > 0.05 + ``` + +- **Transaction Throughput Drop** (warning, `for: 10m`): fires when the `tx.receive` span rate falls below 10/s. + + ``` + {resource.service.name="xrpld" && name="tx.receive"} | rate() < 10 + ``` + +> **Note**: The Consensus Round Slow and Transaction Throughput Drop rules use TraceQL aggregates (`avg(duration)`, `rate()`), which require Tempo 2.3+ with TraceQL metrics enabled. Verify aggregate query support in your Tempo version before provisioning. The RPC Error Rate Spike rule instead queries Prometheus span metrics (collector `spanmetrics` connector), so it needs that connector enabled in the collector pipeline. + +--- + +## 7.7 PerfLog and Insight Correlation + +> **OTLP** = OpenTelemetry Protocol + +How to correlate OpenTelemetry traces with existing xrpld observability. + +### 7.7.1 Correlation Architecture + +```mermaid +flowchart TB + subgraph xrpld["xrpld Node"] + otel[OpenTelemetry
Spans] + perflog[PerfLog
JSON Logs] + insight[Beast Insight
StatsD Metrics] + end + + subgraph collectors["Data Collection"] + otelc[OTel Collector] + promtail[Promtail/Fluentd] + statsd[StatsD Exporter] + end + + subgraph storage["Storage"] + tempo[(Tempo)] + loki[(Loki)] + prom[(Prometheus)] + end + + subgraph grafana["Grafana"] + traces[Trace View] + logs[Log View] + metrics[Metrics View] + corr[Correlation
Panel] + end + + otel -->|OTLP| otelc --> tempo + perflog -->|JSON| promtail --> loki + insight -->|StatsD| statsd --> prom + + tempo --> traces + loki --> logs + prom --> metrics + + traces --> corr + logs --> corr + metrics --> corr + + style xrpld fill:#0d47a1,stroke:#082f6a,color:#fff + style collectors fill:#bf360c,stroke:#8c2809,color:#fff + style storage fill:#1b5e20,stroke:#0d3d14,color:#fff + style grafana fill:#4a148c,stroke:#2e0d57,color:#fff + style otel fill:#0d47a1,stroke:#082f6a,color:#fff + style perflog fill:#0d47a1,stroke:#082f6a,color:#fff + style insight fill:#0d47a1,stroke:#082f6a,color:#fff + style otelc fill:#bf360c,stroke:#8c2809,color:#fff + style promtail fill:#bf360c,stroke:#8c2809,color:#fff + style statsd fill:#bf360c,stroke:#8c2809,color:#fff + style tempo fill:#1b5e20,stroke:#0d3d14,color:#fff + style loki fill:#1b5e20,stroke:#0d3d14,color:#fff + style prom fill:#1b5e20,stroke:#0d3d14,color:#fff + style traces fill:#4a148c,stroke:#2e0d57,color:#fff + style logs fill:#4a148c,stroke:#2e0d57,color:#fff + style metrics fill:#4a148c,stroke:#2e0d57,color:#fff + style corr fill:#4a148c,stroke:#2e0d57,color:#fff +``` + +**Reading the diagram:** + +- **xrpld Node (three sources)**: A single node emits three independent data streams -- OpenTelemetry spans, PerfLog JSON logs, and Beast Insight StatsD metrics. +- **Data Collection layer**: Each stream has its own collector -- OTel Collector for spans, Promtail/Fluentd for logs, and a StatsD exporter for metrics. They operate independently. +- **Storage layer (Tempo, Loki, Prometheus)**: Each data type lands in a purpose-built store optimized for its query patterns (trace search, log grep, metric aggregation). +- **Grafana Correlation Panel**: The key integration point -- Grafana queries all three stores and links them via shared fields (`trace_id`, `tx_hash`, `ledger_seq`), enabling a single-pane debugging experience. + +### 7.7.2 Correlation Fields + +| Source | Field | Link To | Purpose | +| ----------- | ------------------- | ------------- | -------------------------- | +| **Trace** | `trace_id` | Logs | Find log entries for trace | +| **Trace** | `tx_hash` | Logs, Metrics | Find TX-related data | +| **Trace** | `ledger_seq` | Logs | Find ledger-related logs | +| **PerfLog** | `trace_id` (new) | Traces | Jump to trace from log | +| **PerfLog** | `ledger_seq` | Traces | Find consensus trace | +| **Insight** | `exemplar.trace_id` | Traces | Jump from metric spike | + +### 7.7.3 Example: Debugging a Slow Transaction + +**Step 1: Find the trace** + +``` +# In Grafana Explore with Tempo +{resource.service.name="xrpld" && span.tx_hash="ABC123..."} +``` + +**Step 2: Get the trace_id from the trace view** + +``` +Trace ID: 4bf92f3577b34da6a3ce929d0e0e4736 +``` + +**Step 3: Find related PerfLog entries** + +``` +# In Grafana Explore with Loki +{job="xrpld"} |= "4bf92f3577b34da6a3ce929d0e0e4736" +``` + +**Step 4: Check Insight metrics for the time window** + +``` +# In Grafana with Prometheus +rate(xrpld_tx_applied_total[1m]) + @ timestamp_from_trace +``` + +### 7.7.4 Unified Dashboard Example + +A single dashboard (uid `xrpld-unified`) that ties traces, metrics, and logs together across the Tempo, Prometheus, and Loki datasources: + +- **Transaction Latency (Traces)** (timeseries, Tempo): `histogram_over_time(duration)` of `tx.receive` spans. +- **Transaction Rate (Metrics)** (timeseries, Prometheus): `rate(xrpld_tx_received_total[5m])` per instance, with a data link that opens the matching `tx.receive` traces in Tempo. +- **Recent Logs** (logs, Loki): `{job="xrpld"} | json`. +- **Trace Search** (table, Tempo): all `xrpld` traces, with per-row data links on `traceID` that jump to the trace in Tempo and to the correlated logs in Loki (`{job="xrpld"} |= ""`). + +The cross-datasource data links are what make this a single-pane debugging view; the correlation fields they rely on are listed in section 7.7.2. + +--- + +_Previous: [Implementation Phases](./06-implementation-phases.md)_ | _Next: [Appendix](./08-appendix.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/08-appendix.md b/OpenTelemetryPlan/08-appendix.md new file mode 100644 index 00000000000..c75af13d127 --- /dev/null +++ b/OpenTelemetryPlan/08-appendix.md @@ -0,0 +1,196 @@ +# Appendix + +> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) +> **Related**: [Observability Backends](./07-observability-backends.md) + +--- + +## 8.1 Glossary + +> **OTLP** = OpenTelemetry Protocol | **TxQ** = Transaction Queue + +| Term | Definition | +| --------------------- | ---------------------------------------------------------- | +| **Span** | A unit of work with start/end time, name, and attributes | +| **Trace** | A collection of spans representing a complete request flow | +| **Trace ID** | 128-bit unique identifier for a trace | +| **Span ID** | 64-bit unique identifier for a span within a trace | +| **Context** | Carrier for trace/span IDs across boundaries | +| **Propagator** | Component that injects/extracts context | +| **Sampler** | Decides which traces to record | +| **Exporter** | Sends spans to backend | +| **Collector** | Receives, processes, and forwards telemetry | +| **OTLP** | OpenTelemetry Protocol (wire format) | +| **W3C Trace Context** | Standard HTTP headers for trace propagation | +| **Baggage** | Key-value pairs propagated across service boundaries | +| **Resource** | Entity producing telemetry (service, host, etc.) | +| **Instrumentation** | Code that creates telemetry data | + +### xrpld-Specific Terms + +| Term | Definition | +| ----------------- | ------------------------------------------------------------- | +| **Overlay** | P2P network layer managing peer connections | +| **Consensus** | XRP Ledger consensus algorithm (RCL) | +| **Proposal** | Validator's suggested transaction set for a ledger | +| **Validation** | Validator's signature on a closed ledger | +| **HashRouter** | Component for transaction deduplication | +| **JobQueue** | Thread pool for asynchronous task execution | +| **PerfLog** | Existing performance logging system in xrpld | +| **Beast Insight** | Existing metrics framework in xrpld | +| **PathFinding** | Payment path computation engine for cross-currency payments | +| **TxQ** | Transaction queue managing fee-based prioritization | +| **LoadManager** | Dynamic fee escalation based on network load | +| **SHAMap** | SHA-256 hash-based map (Merkle trie variant) for ledger state | + +### Phase 9–11 Terms + +| Term | Definition | +| --------------------------- | ------------------------------------------------------------------------- | +| **MetricsRegistry** | Centralized class for OTel async gauge registrations (Phase 9) | +| **ObservableGauge** | OTel Metrics SDK async instrument polled via callback at fixed intervals | +| **PeriodicMetricReader** | OTel SDK component that invokes gauge callbacks at configurable intervals | +| **CountedObject** | xrpld template that tracks live instance counts via atomic counters | +| **TxQ** | Transaction queue managing fee escalation and ordering | +| **Load Factor** | Combined multiplier affecting transaction cost (local, cluster, network) | +| **OTel Collector Receiver** | Custom Go plugin that polls xrpld RPC and emits OTel metrics (Phase 11) | + +--- + +## 8.2 Span Hierarchy Visualization + +The authoritative span-flow diagrams — a master overview plus per-stage +flowcharts (ingress, the shared apply pipeline, the consensus round, ledger +finalize, and the pathfinding / ledger-acquire side flows) — live in the operator +runbook. They map every span onto the **real xrpld control flow and XRPL protocol +order** (verified against code and `docs/consensus.md`, with file:line evidence), +label every node and branch with the span that represents that state or +transition, and call out where the OpenTelemetry span parent links diverge from +that flow. + +> **See**: [docs/telemetry-runbook.md § Protocol Span Flow](../docs/telemetry-runbook.md#protocol-span-flow). + +The full span inventory (names, attributes, parents as instrumented) is in +[09-data-collection-reference.md §1](./09-data-collection-reference.md#1-opentelemetry-spans). + +--- + +## 8.3 References + +> **OTLP** = OpenTelemetry Protocol + +### OpenTelemetry Resources + +1. [OpenTelemetry C++ SDK](https://github.com/open-telemetry/opentelemetry-cpp) +2. [OpenTelemetry Specification](https://opentelemetry.io/docs/specs/otel/) +3. [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) +4. [OTLP Protocol Specification](https://opentelemetry.io/docs/specs/otlp/) + +### Standards + +5. [W3C Trace Context](https://www.w3.org/TR/trace-context/) +6. [W3C Baggage](https://www.w3.org/TR/baggage/) +7. [Protocol Buffers](https://protobuf.dev/) + +### xrpld Resources + +8. [xrpld Source Code](https://github.com/XRPLF/rippled) +9. [XRP Ledger Documentation](https://xrpl.org/docs/) +10. [xrpld Overlay README](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/README.md) +11. [xrpld RPC README](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/README.md) +12. [xrpld Consensus README](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/README.md) + +--- + +## 8.4 Version History + +| Version | Date | Author | Changes | +| ------- | ---------- | ------ | -------------------------------------------------------------- | +| 1.0 | 2026-02-12 | - | Initial implementation plan | +| 1.1 | 2026-02-13 | - | Refactored into modular documents | +| 1.2 | 2026-03-09 | - | Added Phases 9–11 (future enhancement plans) | +| 1.3 | 2026-03-24 | - | Review fixes: accuracy corrections, cross-document consistency | + +--- + +## 8.5 Document Index + +### Plan Documents + +| Document | Description | +| -------------------------------------------------------------------- | -------------------------------------------------- | +| [OpenTelemetryPlan.md](./OpenTelemetryPlan.md) | Master overview and executive summary | +| [00-tracing-fundamentals.md](./00-tracing-fundamentals.md) | Distributed tracing concepts and OTel primer | +| [01-architecture-analysis.md](./01-architecture-analysis.md) | xrpld architecture and trace points | +| [02-design-decisions.md](./02-design-decisions.md) | SDK selection, exporters, span conventions | +| [03-implementation-strategy.md](./03-implementation-strategy.md) | Directory structure, performance analysis | +| [05-configuration-reference.md](./05-configuration-reference.md) | xrpld config, CMake, Collector configs | +| [06-implementation-phases.md](./06-implementation-phases.md) | Timeline, tasks, risks, success metrics | +| [07-observability-backends.md](./07-observability-backends.md) | Backend selection and architecture | +| [08-appendix.md](./08-appendix.md) | Glossary, references, version history | +| [secure-OTel.md](./secure-OTel.md) | Threat model and hardening (mTLS, peer validation) | +| [09-data-collection-reference.md](./09-data-collection-reference.md) | Span/metric/dashboard inventory | + +### Task Lists + +| Document | Description | +| -------------------------------------------------------------------------- | --------------------------------------------------- | +| [Phase2_taskList.md](./Phase2_taskList.md) | RPC layer trace instrumentation | +| [Phase3_taskList.md](./Phase3_taskList.md) | Peer overlay & consensus tracing | +| [Phase4_taskList.md](./Phase4_taskList.md) | Transaction lifecycle tracing | +| [Phase5_taskList.md](./Phase5_taskList.md) | Ledger processing & advanced tracing | +| [Phase5_IntegrationTest_taskList.md](./Phase5_IntegrationTest_taskList.md) | Observability stack integration tests | +| [Phase7_taskList.md](./Phase7_taskList.md) | Native OTel metrics migration | +| [Phase8_taskList.md](./Phase8_taskList.md) | Log-trace correlation | +| [Phase9_taskList.md](./Phase9_taskList.md) | Internal metric instrumentation gap fill (future) | +| [Phase10_taskList.md](./Phase10_taskList.md) | Synthetic workload generation & validation (future) | +| [Phase11_taskList.md](./Phase11_taskList.md) | Third-party data collection pipelines (future) | + +> **Note**: Phases 1 and 6 do not have separate task list files. Phase 1 tasks are documented in [06-implementation-phases.md §6.2](./06-implementation-phases.md). Phase 6 tasks are documented in [06-implementation-phases.md §6.7](./06-implementation-phases.md). + +--- + +## 8.6 Phase 9–11 Cross-Reference Guide + +This guide maps Phase 9–11 content to its location across the documentation. + +### Phase 9: Internal Metric Instrumentation Gap Fill + +| Content | Location | +| ------------------------------- | ------------------------------------------------------------------------ | +| Plan & architecture | [06-implementation-phases.md §6.8.2](./06-implementation-phases.md) | +| Task list (10 tasks) | [Phase9_taskList.md](./Phase9_taskList.md) | +| Future metric definitions (~50) | [09-data-collection-reference.md §5b](./09-data-collection-reference.md) | +| New class: `MetricsRegistry` | `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (planned) | +| New dashboards | `fee-market`, `job-queue` (planned) | + +**Metric categories**: NodeStore I/O, Cache Hit Rates, TxQ, PerfLog Per-RPC, PerfLog Per-Job, Counted Objects, Fee Escalation & Load Factors. + +### Phase 10: Synthetic Workload Generation & Telemetry Validation + +| Content | Location | +| -------------------- | ------------------------------------------------------------------------ | +| Plan & architecture | [06-implementation-phases.md §6.8.3](./06-implementation-phases.md) | +| Task list (7 tasks) | [Phase10_taskList.md](./Phase10_taskList.md) | +| Validation inventory | [09-data-collection-reference.md §5c](./09-data-collection-reference.md) | +| Test harness | `docker/telemetry/docker-compose.workload.yaml` (planned) | +| CI workflow | `.github/workflows/telemetry-validation.yml` (planned) | + +**Validates**: 16 spans, 22 attributes, 300+ metrics, 10 dashboards, log-trace correlation. + +### Phase 11: Third-Party Data Collection Pipelines + +| Content | Location | +| --------------------------------- | ------------------------------------------------------------------------ | +| Plan & architecture | [06-implementation-phases.md §6.8.4](./06-implementation-phases.md) | +| Task list (11 tasks) | [Phase11_taskList.md](./Phase11_taskList.md) | +| External metric definitions (~30) | [09-data-collection-reference.md §5d](./09-data-collection-reference.md) | +| Custom OTel Collector receiver | `docker/telemetry/otel-rippled-receiver/` (planned) | +| Prometheus alerting rules (11) | [09-data-collection-reference.md §5d](./09-data-collection-reference.md) | +| New dashboards (4) | Validator Health, Network Topology, Fee Market (External), DEX & AMM | + +**Consumer categories**: Exchanges, Payment Processors, DeFi/AMM, NFT Marketplaces, Analytics Providers, Wallets, Compliance, Academic Researchers, Institutional Custody, CBDC Bridge Operators. + +--- + +_Previous: [Observability Backends](./07-observability-backends.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md new file mode 100644 index 00000000000..57090ee77ec --- /dev/null +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -0,0 +1,1613 @@ +# Observability Data Collection Reference + +> **Audience**: Developers and operators. This is the single source of truth for all telemetry data collected by xrpld's observability stack. +> +> **Related docs**: [docs/telemetry-runbook.md](../docs/telemetry-runbook.md) (operator runbook with alerting and troubleshooting) | [03-implementation-strategy.md](./03-implementation-strategy.md) (code structure and performance optimization) | [04-code-samples.md](./04-code-samples.md) (C++ instrumentation examples) + +## Data Flow Overview + +```mermaid +graph LR + subgraph xrpldNode["xrpld Node"] + A["Trace Macros
XRPL_TRACE_SPAN
(OTLP/HTTP exporter)"] + B["beast::insight
OTel native metrics
(OTLP/HTTP exporter)"] + C["MetricsRegistry
OTel SDK metrics
(OTLP/HTTP exporter)"] + end + + subgraph collector["OTel Collector :4317 / :4318"] + direction TB + R1["OTLP Receiver
:4317 gRPC | :4318 HTTP
(traces + metrics)"] + BP["Batch Processor
timeout 1s, batch 100"] + SM["SpanMetrics Connector
derives RED metrics
from trace spans"] + + R1 --> BP + BP --> SM + end + + subgraph backends["Trace Backend"] + D["Grafana Tempo :3200
TraceQL search &
S3/GCS long-term storage"] + end + + subgraph metrics["Metrics Stack"] + E["Prometheus :9090
scrapes :8889
span-derived + system metrics"] + end + + subgraph viz["Visualization"] + F["Grafana :3000
13 dashboards"] + end + + A -->|"OTLP/HTTP :4318
(traces + attributes)"| R1 + B -->|"OTLP/HTTP :4318
(gauges, counters, histograms)"| R1 + C -->|"OTLP/HTTP :4318
(counters, histograms,
observable gauges)"| R1 + + BP -->|"OTLP/gRPC :4317"| D + + SM -->|"span_calls_total
span_duration_ms
(6 dimension labels)"| E + R1 -->|"gauges, counters,
histograms (OTLP)"| E + + E -->|"Prometheus
data source"| F + D -->|"Tempo
data source"| F + + style A fill:#4a90d9,color:#fff,stroke:#2a6db5 + style B fill:#4a90d9,color:#fff,stroke:#2a6db5 + style C fill:#4a90d9,color:#fff,stroke:#2a6db5 + style R1 fill:#5cb85c,color:#fff,stroke:#3d8b3d + style BP fill:#449d44,color:#fff,stroke:#2d6e2d + style SM fill:#449d44,color:#fff,stroke:#2d6e2d + style D fill:#f0ad4e,color:#000,stroke:#c78c2e + style E fill:#f0ad4e,color:#000,stroke:#c78c2e + style F fill:#5bc0de,color:#000,stroke:#3aa8c1 + style xrpldNode fill:#1a2633,color:#ccc,stroke:#4a90d9 + style collector fill:#1a3320,color:#ccc,stroke:#5cb85c + style backends fill:#332a1a,color:#ccc,stroke:#f0ad4e + style metrics fill:#332a1a,color:#ccc,stroke:#f0ad4e + style viz fill:#1a2d33,color:#ccc,stroke:#5bc0de +``` + +There are three independent telemetry pipelines entering a single **OTel Collector** via the same OTLP receiver — nodes **A**, **B**, and **C** in the diagram above: + +1. **OpenTelemetry Traces** (**A**) — Distributed spans with attributes, exported via OTLP/HTTP (:4318) to the collector's **OTLP Receiver**. The **Batch Processor** groups spans (1s timeout, batch size 100) before forwarding to trace backends. The **SpanMetrics Connector** derives RED metrics (rate, errors, duration) from every span and feeds them into the metrics pipeline. +2. **beast::insight OTel Metrics** (**B**) — System-level gauges, counters, and histograms exported natively via OTLP/HTTP (:4318) to the same **OTLP Receiver**. These are batched and exported to Prometheus alongside span-derived metrics. The StatsD UDP transport has been replaced by native OTLP; `server=statsd` remains available as a fallback. +3. **MetricsRegistry OTel SDK Metrics** (**C**) — Counters, histograms, and observable gauges registered directly with the OTel Metrics SDK, exported via OTLP/HTTP (:4318). This pipeline owns its own `MeterProvider` and reader, separate from **B**'s, so its export cadence is independent — see [§2.5](#25-per-job-type-queue-gauges) for why that matters when comparing the two. + +A third, narrower metrics path exists for instruments created at their call site through the +`XRPL_METRIC_*` macros. These use the OTel Metrics SDK directly and reach the collector's OTLP +receiver rather than the StatsD receiver, so their names carry no `xrpld_` prefix. See +[§2a](#2a-call-site-otel-metrics-metricsregistry). Code in `libxrpl` cannot use these macros and +always goes through `beast::insight` instead. + +**Trace backend** — The collector exports traces via OTLP/gRPC to: + +- **Grafana Tempo** — Preferred trace backend. Supports TraceQL queries at `:3200`, S3/GCS object storage for cost-effective long-term trace retention, and integrates natively with Grafana. + +> **Further reading**: [00-tracing-fundamentals.md](./00-tracing-fundamentals.md) for core OpenTelemetry concepts (traces, spans, context propagation, sampling). [07-observability-backends.md](./07-observability-backends.md) for production backend selection, collector placement, and sampling strategies. + +--- + +## 1. OpenTelemetry Spans + +### 1.1 Complete Span Inventory (~37 spans) + +> **See also**: [02-design-decisions.md §2.3](./02-design-decisions.md#23-span-naming-conventions) for naming conventions and the full span catalog with rationale. [04-code-samples.md §4.6](./04-code-samples.md#46-span-flow-visualization) for span flow diagrams. + +> **Span names vs. attribute keys**: span names use dotted `subsystem.operation` +> form (e.g. `rpc.http_request`). Span _attribute_ keys use the bare/underscore +> form from the 2026-05-13 naming redesign (e.g. `tx_hash`, not `xrpl.tx.hash`). +> The dotted `xrpl.*` form is reserved for OTel **resource** attributes set once +> at startup. See §1.2 for the full attribute inventory. + +#### RPC Spans + +Controlled by `trace_rpc=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| -------------------- | ------------------ | ----------------- | ------------------------------------------------------------------------ | +| `rpc.http_request` | — | ServerHandler.cpp | Top-level HTTP JSON-RPC request entry point | +| `rpc.ws_message` | — | ServerHandler.cpp | WebSocket message handling (one per inbound frame) | +| `rpc.ws_upgrade` | — | ServerHandler.cpp | WebSocket upgrade handshake (records handshake failures) | +| `rpc.process` | `rpc.http_request` | ServerHandler.cpp | RPC processing pipeline (single or batch request) | +| `rpc.command.` | `rpc.process` | RPCHandler.cpp | Per-command span (e.g., `rpc.command.server_info`, `rpc.command.ledger`) | + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"rpc.http_request|rpc.command.*"}` + +**Grafana dashboard**: _RPC Performance_ (`rpc-performance`) + +#### gRPC Spans + +Controlled by `trace_rpc=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| ------------------- | ------ | -------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `grpc.` | — | GRPCServer.cpp | One flat span per gRPC method (e.g., `grpc.GetLedger`, `grpc.GetLedgerData`, `grpc.GetLedgerDiff`, `grpc.GetLedgerEntry`) | + +The method name is embedded in the span name (formed at the call site as +`grpc.`), so dashboards break out per-method latency and error +rates without TraceQL attribute filters. + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"grpc.*"}` + +**Grafana dashboard**: _RPC Performance_ (`rpc-performance`) + +#### Transaction Spans + +Controlled by `trace_transactions=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| --------------- | -------------- | --------------- | ----------------------------------------------------------------- | +| `tx.process` | — | NetworkOPs.cpp | Transaction submission entry point (local or peer-relayed) | +| `tx.receive` | — | PeerImp.cpp | Raw transaction received from peer overlay (before deduplication) | +| `tx.apply` | `ledger.build` | BuildLedger.cpp | Transaction set applied to new ledger during consensus | +| `tx.preflight` | — | applySteps.cpp | Stateless checks stage (`stage=preflight`) | +| `tx.preclaim` | — | applySteps.cpp | Ledger-aware checks stage before fee claim (`stage=preclaim`) | +| `tx.transactor` | — | Transactor.cpp | Apply stage — the transactor runs (`stage=apply`) | + +The three apply-pipeline spans share a deterministic `trace_id` derived from +`txID[0:16]`, so preflight, preclaim, and transactor for one transaction group +under a single trace even though they run sequentially and often on different +threads. A transaction that hard-fails preflight or preclaim never reaches the +later spans — the `stage` attribute identifies where it stopped. + +> **Deterministic roots are true roots.** Spans with a deterministic `trace_id` +> (the `tx.*` apply pipeline, `tx.process`, `tx.receive`, and `consensus.round`) +> are emitted as genuine trace roots with an empty `parent_span_id`. The chosen +> `trace_id` is injected through a custom `DeterministicIdGenerator` on the SDK's +> no-parent branch, so there is no synthetic placeholder parent — Tempo shows a +> clean root, not a "root span not yet received" warning. Cross-node correlation +> still works because every node derives the same `trace_id` from the shared hash. + +> **Log-trace correlation is retained across coroutines.** OTel context storage +> is coroutine-aware (backed by `LocalValue`), so the active span travels with a +> coroutine across `yield()` and resumes on whatever thread the scheduler picks. +> RPC, consensus, and transaction spans therefore keep per-line log-trace +> correlation, and their scopes are safe across coroutine yields. Job-handoff +> spans — transaction apply and receive, consensus accept, and ledger acquire — +> are activated inside their worker bodies rather than at enqueue, so each +> worker's log lines carry the span's trace context. + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"tx.process|tx.receive"}` +or, for the apply pipeline: `{resource.service.name="xrpld" && name=~"tx.preflight|tx.preclaim|tx.transactor"}` + +**Grafana dashboard**: _Transaction Overview_ (`transaction-overview`) + +#### Transaction Queue (TxQ) Spans + +Controlled by `trace_transactions=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| ------------------ | ----------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `txq.enqueue` | `tx.process` (submission path; root on open-ledger rebuild) | TxQ.cpp | Queue admission decision (apply/queue/reject). Parents to `tx.process` via explicit context on submit; correlates via `current_ledger_seq` on all paths | +| `txq.apply_direct` | `txq.enqueue` | TxQ.cpp | Direct apply attempt that bypasses the queue | +| `txq.batch_clear` | `txq.enqueue` | TxQ.cpp | Batch clear of an account's queued txs | +| `txq.accept` | — | TxQ.cpp | Ledger-close accept loop (drains the queue) | +| `txq.accept_tx` | `txq.accept` | TxQ.cpp | Per-queued-transaction apply inside the accept loop | +| `txq.cleanup` | — | TxQ.cpp | Post-close cleanup of expired queue entries | + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"txq.*"}` + +**Grafana dashboard**: _Transaction Overview_ (`transaction-overview`) + +#### Consensus Spans + +Controlled by `trace_consensus=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| ------------------------------ | ------------------ | ---------------- | ------------------------------------------------------------------- | +| `consensus.round` | — (root) | RCLConsensus.cpp | Root span for one consensus round (deterministic trace per round) | +| `consensus.phase.open` | `consensus.round` | Consensus.h | Open phase — collecting transactions before close | +| `consensus.proposal.send` | `consensus.round` | RCLConsensus.cpp | Node broadcasts its transaction set proposal | +| `consensus.ledger_close` | `consensus.round` | RCLConsensus.cpp | Ledger close event triggered by consensus | +| `consensus.establish` | `consensus.round` | Consensus.h | Establish phase — converging on the transaction set | +| `consensus.update_positions` | `consensus.round` | Consensus.h | Position update with per-dispute vote details | +| `consensus.check` | `consensus.round` | Consensus.h | Consensus threshold check (agree/disagree tally) | +| `consensus.accept` | `consensus.round` | RCLConsensus.cpp | Consensus accepts a ledger (round complete) | +| `consensus.accept.apply` | `consensus.accept` | RCLConsensus.cpp | Ledger application with close-time details (jtACCEPT thread) | +| `consensus.validation.send` | `consensus.round` | RCLConsensus.cpp | Validation message sent after ledger accepted (follows-from link) | +| `consensus.mode_change` | `consensus.round` | RCLConsensus.cpp | Operating-mode transition during the round | +| `consensus.proposal.receive` | (context) | PeerImp.cpp | Proposal received from a peer (context-propagated into the round) | +| `consensus.validation.receive` | (context) | PeerImp.cpp | Validation received from a peer (context-propagated into the round) | + +The `.receive` spans are created per-message in the overlay and joined to the +round trace via context propagation rather than direct parenting. The +`consensus.validation.send` span uses a follows-from link off the round. + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"consensus.*"}` + +**Grafana dashboard**: _Consensus Health_ (`consensus-health`) + +#### Ledger Spans + +Controlled by `trace_ledger=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| ----------------- | ------ | ----------------- | ---------------------------------------------- | +| `ledger.build` | — | BuildLedger.cpp | Build new ledger from accepted transaction set | +| `ledger.validate` | — | LedgerMaster.cpp | Ledger promoted to validated status | +| `ledger.store` | — | LedgerMaster.cpp | Ledger stored to database/history | +| `ledger.acquire` | — | InboundLedger.cpp | Fetch a missing ledger from peers | + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"ledger.*"}` + +**Grafana dashboard**: _Ledger Operations_ (`ledger-operations`) + +#### Peer Spans + +Controlled by `trace_peer` in `[telemetry]` config. **Enabled by default** (high volume). + +| Span Name | Parent | Source File | Description | +| ------------------------- | ------ | ----------- | ------------------------------------- | +| `peer.proposal.receive` | — | PeerImp.cpp | Consensus proposal received from peer | +| `peer.validation.receive` | — | PeerImp.cpp | Validation message received from peer | + +A `—` parent means the span is a fresh trace root (`kConsumer`): it is started +via `ScopedSpanGuard::freshRoot()` at the inbound-message entry point and never +inherits an ambient span left active on the peer thread, so it does not nest +under an unrelated transaction's trace. + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"peer.*"}` + +**Grafana dashboard**: _Peer Network_ (`peer-network`) + +#### PathFind Spans + +Controlled by `trace_rpc=1` in `[telemetry]` config. + +| Span Name | Parent | Source File | Description | +| --------------------- | -------------------- | --------------- | ---------------------------------------------------------- | +| `pathfind.request` | `rpc.command.` | PathFind.cpp | `path_find` RPC entry (`doPathFind`) | +| `pathfind.compute` | `pathfind.request` | PathRequest.cpp | Path computation for one request (`PathRequest::doUpdate`) | +| `pathfind.discover` | `pathfind.compute` | Pathfinder.cpp | Graph exploration (one per RPC call) | +| `pathfind.update_all` | — | PathRequest.cpp | Async recomputation of all active requests at ledger close | + +> **Note**: `pathfind.request` nests under the active `rpc.command.` span. +> Because OTel context storage is coroutine-aware (backed by `LocalValue`), the +> `rpc.command.*` scope stays correct even though its generic dispatch +> (`callMethod`) also wraps handlers such as `doRipplePathFind` whose span is held +> across a coroutine `yield()` — the ambient context travels with the coroutine +> when it resumes, so there is no wrong-thread scope pop. The +> `pathfind.request → compute → discover` sub-tree therefore parents to +> `rpc.command.`, giving an exact request-to-pathfind nesting. + +**Where to find**: Tempo → TraceQL: `{resource.service.name="xrpld" && name=~"pathfind.*"}` + +--- + +### 1.2 Complete Attribute Inventory (bare/underscore keys) + +> **See also**: [02-design-decisions.md §2.4.2](./02-design-decisions.md#242-span-attributes-by-category) for attribute design rationale and privacy considerations. + +Every span can carry key-value attributes that provide context for filtering and +aggregation. Per the 2026-05-13 naming redesign, span-attribute keys use the +**bare** field name (the span name already carries the domain), or the +`_` underscore form where a bare name would collide (e.g. +`rpc_status`, `grpc_status`, `tx_status`, `txq_status`). + +> **Dotted keys are resource attributes, never span attributes:** +> +> - `xrpl.network.id` and `xrpl.network.type` are **resource** attributes set +> once at startup on the OTel resource — not span attributes. They appear on +> every span's resource scope, queried as `{resource.xrpl.network.id=...}`. +> - The ledger hash uses the bare `ledger_hash` key on every span that records +> it (both `consensus.validation.send` and `peer.validation.receive`) — there +> is no dotted span attribute. + +#### RPC Attributes + +| Attribute | Type | Set On | Description | +| ---------------------- | ------- | --------------------------------- | ------------------------------------------------ | +| `command` | string | `rpc.command.*`, `rpc.ws_message` | RPC command name (e.g., `server_info`, `ledger`) | +| `version` | int64 | `rpc.command.*` | API version number | +| `rpc_role` | string | `rpc.command.*` | Caller role: `"admin"` or `"user"` | +| `rpc_status` | string | `rpc.command.*` | Result: `"success"` or `"error"` | +| `request_payload_size` | int64 | `rpc.http_request` | Bytes of inbound request payload | +| `is_batch` | boolean | `rpc.process` | `true` if the request is a JSON-RPC batch | +| `batch_size` | int64 | `rpc.process` | Number of sub-requests in a batch | +| `load_type` | string | `rpc.command.*` | Resource cost category after execution | + +**Tempo query**: `{span.command="server_info"}` to find all `server_info` calls. + +**Prometheus label**: `command` (used as a SpanMetrics dimension). + +#### gRPC Attributes + +| Attribute | Type | Set On | Description | +| ------------- | ------ | ------------------- | ------------------------------------ | +| `method` | string | `grpc.` | gRPC method name (e.g., `GetLedger`) | +| `grpc_role` | string | `grpc.` | Caller role: `"admin"` or `"user"` | +| `grpc_status` | string | `grpc.` | Result: `"success"` or `"error"` | + +**Tempo query**: `{span.method="GetLedger"}` or `{name="grpc.GetLedger"}`. + +**Prometheus labels**: `method`, `grpc_role`, `grpc_status` (SpanMetrics dimensions). + +#### Transaction Attributes + +| Attribute | Type | Set On | Description | +| --------------------- | ------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------- | +| `tx_hash` | string | `tx.process`, `tx.receive` | Transaction hash (hex-encoded) | +| `local` | boolean | `tx.process` | `true` if locally submitted, `false` if peer-relayed | +| `path` | string | `tx.process` | Submission path: `"sync"` or `"async"` | +| `tx_type` | string | `tx.process`, `tx.preflight`, `tx.preclaim`, `tx.transactor` | Transaction type name (e.g., `Payment`) | +| `fee` | int64 | `tx.process` | Transaction fee in drops | +| `sequence` | int64 | `tx.process` | Transaction sequence number | +| `suppressed` | boolean | `tx.receive` | `true` if transaction was suppressed (duplicate) | +| `tx_status` | string | `tx.receive` | Transaction status (e.g., `"known_bad"`) | +| `peer_id` | int64 | `tx.receive` | Peer identifier (also set on peer spans) | +| `peer_version` | string | `tx.receive` | Peer protocol version string | +| `stage` | string | `tx.preflight`, `tx.preclaim`, `tx.transactor` | Apply-pipeline stage: `preflight`, `preclaim`, or `apply` | +| `ter_result` | string | `tx.preflight`, `tx.preclaim`, `tx.transactor` | Engine result token for that stage (e.g., `tesSUCCESS`, `terPRE_SEQ`) | +| `applied` | boolean | `tx.transactor` | `true` if the transaction was applied to the ledger | +| `current_ledger_seq` | int64 | `tx.process`, `tx.receive`, `tx.preclaim`, `tx.transactor` | Seq of the ledger being worked on (open/in-flight, not established) — joins the txID-keyed spans to the ledger trace | +| `current_ledger_hash` | string | `tx.preclaim`, `tx.transactor` | Parent hash of that ledger (= `consensus.round` trace-id seed on the build path). View-bearing stages only; `tx.preflight` omits both | + +**Tempo query**: `{span.tx_hash=""}` to trace a specific transaction across nodes. +Join a transaction's work to its ledger with `{span.current_ledger_seq=}`. + +**Prometheus labels**: `local`, `suppressed`, `tx_type`, `ter_result`, `stage` (SpanMetrics dimensions). + +#### Transaction Queue (TxQ) Attributes + +| Attribute | Type | Set On | Description | +| --------------------- | ------- | ------------------------------ | -------------------------------------------------------------------------------- | +| `tx_hash` | string | `txq.enqueue`, `txq.accept_tx` | Transaction hash | +| `tx_type` | string | `txq.enqueue` | Transaction type name | +| `current_ledger_seq` | int64 | `txq.enqueue` | Seq of the ledger being worked on — correlates the enqueue to the ledger trace | +| `current_ledger_hash` | string | `txq.enqueue` | Parent hash of that ledger (= `consensus.round` trace-id seed on the build path) | +| `txq_status` | string | `txq.enqueue`, `txq.accept_tx` | Queue outcome (e.g. `queued`, `applied_direct`, `rejected`) | +| `fee_level_paid` | int64 | `txq.enqueue` | Fee level paid by the queued tx | +| `required_fee_level` | int64 | `txq.enqueue` | Minimum fee level for inclusion | +| `num_cleared` | int64 | `txq.batch_clear` | Entries cleared in a batch | +| `queue_size` | int64 | `txq.accept` | Current TxQ depth | +| `ledger_changed` | boolean | `txq.accept` | Whether the ledger changed since last attempt | +| `ter_code` | int64 | `txq.accept_tx` | Transaction engine result code | +| `retries_remaining` | int64 | `txq.accept_tx` | Retries left before discard | +| `ledger_seq` | int64 | `txq.cleanup` | Ledger sequence number | +| `expired_count` | int64 | `txq.cleanup` | Number of expired entries cleared | + +**Prometheus label**: `txq_status` (SpanMetrics dimension). + +#### Consensus Attributes + +| Attribute | Type | Set On | Description | +| -------------------------- | ------- | -------------------------------------------------------------------------------------------------- | -------------------------------------------------------- | +| `consensus_ledger_id` | string | `consensus.round` | Previous-ledger id anchoring the round | +| `ledger_seq` | int64 | `consensus.round`, `consensus.ledger_close`, `consensus.accept.apply`, `consensus.validation.send` | Ledger sequence number | +| `consensus_mode` | string | `consensus.round`, `consensus.ledger_close` | Node mode: `"Proposing"`, `"Observing"`, `"Wrong"`, etc. | +| `consensus_round_id` | int64 | `consensus.round` | Round identifier | +| `consensus_phase` | string | `consensus.round` | Current phase name (updated on each transition) | +| `trace_strategy` | string | `consensus.round` | Trace-id strategy (`deterministic` / `random`) | +| `previous_ledger_seq` | int64 | `consensus.round` | Sequence of the previous ledger | +| `previous_proposers` | int64 | `consensus.round` | Proposer count in the previous round | +| `previous_round_time_ms` | int64 | `consensus.round` | Duration of the previous round | +| `consensus_round` | int64 | `consensus.proposal.send` | Proposal sequence number for the broadcast proposal | +| `is_bow_out` | boolean | `consensus.proposal.send` | Whether the proposal is a bow-out (resigning the round) | +| `tx_count_open` | int64 | `consensus.ledger_close` | Transactions in the open ledger at close | +| `close_time_resolution_ms` | int64 | `consensus.ledger_close` | Close-time rounding granularity | +| `converge_percent` | int64 | `consensus.establish`, `consensus.update_positions` | Convergence percentage | +| `establish_count` | int64 | `consensus.establish` | Establish-phase iteration count | +| `proposers` | int64 | `consensus.establish`, `consensus.update_positions`, `consensus.accept` | Number of proposers | +| `disputes_count` | int64 | `consensus.establish`, `consensus.update_positions` | Number of disputed transactions | +| `tx_id` | string | `consensus.update_positions` | Disputed transaction id (per-dispute event) | +| `dispute_our_vote` | boolean | `consensus.update_positions` | Our vote on the disputed tx | +| `dispute_yays` | int64 | `consensus.update_positions` | Yes votes on the disputed tx | +| `dispute_nays` | int64 | `consensus.update_positions` | No votes on the disputed tx | +| `agree_count` | int64 | `consensus.check` | Agreeing proposer count | +| `disagree_count` | int64 | `consensus.check` | Disagreeing proposer count | +| `threshold_percent` | int64 | `consensus.check` | Agreement threshold percentage | +| `consensus_result` | string | `consensus.check` | Check outcome | +| `quorum` | int64 | `consensus.check`, `consensus.accept` | Quorum required | +| `round_time_ms` | int64 | `consensus.accept`, `consensus.accept.apply` | Total consensus round duration in milliseconds | +| `consensus_state` | string | `consensus.accept.apply` | Consensus outcome: `"finished"` or `"moved_on"` | +| `close_time` | int64 | `consensus.accept.apply` | Agreed-upon ledger close time (epoch seconds) | +| `close_time_correct` | boolean | `consensus.accept.apply` | Whether validators agreed on close time | +| `close_resolution_ms` | int64 | `consensus.accept.apply` | Close-time rounding granularity in milliseconds | +| `proposing` | boolean | `consensus.accept.apply`, `consensus.validation.send` | Whether this node was a proposer | +| `parent_close_time` | int64 | `consensus.accept.apply` | Parent ledger close time | +| `close_time_self` | int64 | `consensus.accept.apply` | This node's close-time vote | +| `close_time_vote_bins` | string | `consensus.accept.apply` | Distribution of close-time votes | +| `resolution_direction` | string | `consensus.accept.apply` | Whether close resolution increased/decreased/unchanged | +| `tx_count` | int64 | `consensus.accept.apply` | Transactions in the accepted set | +| `ledger_hash` | string | `consensus.validation.send` | Full hash of the validated ledger (shared with peer) | +| `full_validation` | boolean | `consensus.validation.send` | Whether this is a full validation | +| `validation_sign_time` | int64 | `consensus.validation.send` | Validation signing time | +| `mode_old` | string | `consensus.mode_change` | Operating mode before the transition | +| `mode_new` | string | `consensus.mode_change` | Operating mode after the transition | + +**Tempo query**: `{span.consensus_mode="Proposing"}` to find rounds where the node was proposing. + +**Prometheus labels**: `consensus_mode`, `consensus_state`, `consensus_phase`, `consensus_result`, `consensus_stalled`, `mode_new`, `close_time_correct` (SpanMetrics dimensions). + +#### Ledger Attributes + +| Attribute | Type | Set On | Description | +| --------------------- | ------- | ------------------------------------------------- | ------------------------------------------------ | +| `ledger_seq` | int64 | `ledger.build`, `ledger.validate`, `ledger.store` | Ledger sequence number | +| `close_time` | int64 | `ledger.build` | Ledger close time (epoch seconds) | +| `close_time_correct` | boolean | `ledger.build` | Whether close time was agreed upon by validators | +| `close_resolution_ms` | int64 | `ledger.build` | Close time rounding granularity in milliseconds | +| `tx_count` | int64 | `tx.apply` | Transactions applied to the ledger | +| `tx_failed` | int64 | `tx.apply` | Failed transactions in the apply set | +| `validations` | int64 | `ledger.validate` | Number of validations received for this ledger | +| `acquire_reason` | string | `ledger.acquire` | Why the ledger fetch was triggered | +| `timeouts` | int64 | `ledger.acquire` | Number of fetch timeouts | +| `peer_count` | int64 | `ledger.acquire` | Peers queried during the fetch | +| `outcome` | string | `ledger.acquire` | Fetch outcome | + +The apply-step span `tx.apply` (child of `ledger.build`) carries `tx_count`/`tx_failed`; +the parent `ledger.build` carries `ledger_seq` and the close-time attributes. +`ledger.acquire` (InboundLedger) also sets `ledger_seq`. + +**Tempo query**: `{span.ledger_seq=12345}` to find all spans for a specific ledger. + +#### Peer Attributes + +| Attribute | Type | Set On | Description | +| -------------------- | ------- | ---------------------------------------------------------------- | ---------------------------------------------------- | +| `peer_id` | int64 | `tx.receive`, `peer.proposal.receive`, `peer.validation.receive` | Peer identifier | +| `proposal_trusted` | boolean | `peer.proposal.receive` | Whether the proposal came from a trusted validator | +| `validation_trusted` | boolean | `peer.validation.receive` | Whether the validation came from a trusted validator | +| `full_validation` | boolean | `peer.validation.receive` | Whether the validation is a full validation | +| `ledger_hash` | string | `peer.validation.receive` | Validated ledger hash (shared with consensus spans) | + +**Prometheus labels**: `proposal_trusted`, `validation_trusted` (SpanMetrics dimensions). + +#### PathFind Attributes + +| Attribute | Type | Set On | Description | +| ------------------------- | ------- | --------------------- | ---------------------------------------- | +| `pathfind_source_account` | string | `pathfind.request` | Originating account for the path search | +| `pathfind_dest_account` | string | `pathfind.request` | Destination account | +| `pathfind_fast` | boolean | `pathfind.compute` | Whether fast pathfinding mode is enabled | +| `pathfind_search_level` | int64 | `pathfind.discover` | Depth of graph exploration | +| `pathfind_num_paths` | int64 | `pathfind.discover` | Total paths produced | +| `pathfind_ledger_index` | int64 | `pathfind.update_all` | Target ledger index | +| `pathfind_num_requests` | int64 | `pathfind.update_all` | Active requests recomputed | + +--- + +### 1.3 SpanMetrics — Derived Prometheus Metrics + +> **See also**: [01-architecture-analysis.md](./01-architecture-analysis.md) §1.8.2 for how span-derived metrics map to operational insights. + +The OTel Collector's SpanMetrics connector automatically generates RED (Rate, Errors, Duration) metrics from every span. No custom metrics code in xrpld is needed. + +| Prometheus Metric | Type | Description | +| ----------------------------------- | --------- | ------------------------------------------------------------------------------ | +| `span_calls_total` | Counter | Total span invocations | +| `span_duration_milliseconds_bucket` | Histogram | Latency distribution (buckets: 1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000 ms) | +| `span_duration_milliseconds_count` | Histogram | Observation count | +| `span_duration_milliseconds_sum` | Histogram | Cumulative latency | + +**Standard labels on every metric**: `span_name`, `status_code`, `service_name`, `span_kind` + +**Additional dimension labels** (configured in `otel-collector-config.yaml`). +The Prometheus label is the **bare span-attribute key verbatim** — the +SpanMetrics connector does not rewrite or prefix it: + +| Prometheus Label / Span Attribute | Type | Applies To | +| --------------------------------- | ------- | ---------------------------------------------- | +| `command` | string | `rpc.command.*` | +| `rpc_status` | string | `rpc.command.*` | +| `consensus_mode` | string | `consensus.round`, `consensus.ledger_close` | +| `close_time_correct` | boolean | `consensus.accept.apply` | +| `local` | boolean | `tx.process` | +| `suppressed` | boolean | `tx.receive` | +| `proposal_trusted` | boolean | `peer.proposal.receive` | +| `validation_trusted` | boolean | `peer.validation.receive` | +| `tx_type` | string | `tx.*`, `txq.enqueue` | +| `ter_result` | string | `tx.preflight`, `tx.preclaim`, `tx.transactor` | +| `stage` | string | `tx.preflight`, `tx.preclaim`, `tx.transactor` | +| `txq_status` | string | `txq.enqueue`, `txq.accept_tx` | +| `consensus_state` | string | `consensus.accept.apply` | +| `load_type` | string | `rpc.command.*` | +| `is_batch` | boolean | `rpc.process` | +| `mode_new` | string | `consensus.mode_change` | +| `consensus_stalled` | boolean | `consensus.check` | +| `consensus_phase` | string | `consensus.round` | +| `consensus_result` | string | `consensus.check` | +| `method` | string | `grpc.` | +| `grpc_role` | string | `grpc.` | +| `grpc_status` | string | `grpc.` | + +The `stage` dimension (3 values: `preflight`, `preclaim`, `apply`) turns the +apply-pipeline spans into per-stage RED metrics with no native instruments — the +_Transaction Overview_ dashboard charts rate, p95 latency, and failure rate by stage. + +> **Sampling caveat**: xrpld head sampling is fixed at 1.0 (every trace is +> recorded), so span-derived metrics are not undercounted at the node. If the +> collector is configured with tail sampling, span-derived metrics reflect only +> the retained traces, whereas native StatsD/meter metrics do not sample. +> Account for any collector-side tail sampling when reading absolute stage rates. + +**Where to query**: Prometheus → `span_calls_total{span_name="rpc.command.server_info"}` + +--- + +## 2. System Metrics (beast::insight — OTel native) + +> **See also**: [02-design-decisions.md](./02-design-decisions.md) for the beast::insight coexistence design. [06-implementation-phases.md](./06-implementation-phases.md) for the Phase 6/7 metric inventory. +> +> **Migration complete**: Phase 7 replaced the StatsD UDP transport with native OTel Metrics SDK export via OTLP/HTTP. The `beast::insight::Collector` interface and all metric names are preserved — only the wire protocol changed. `[insight] server=statsd` remains as a fallback. + +These are system-level metrics emitted by xrpld's `beast::insight` framework via OTel OTLP/HTTP. They cover operational data that doesn't map to individual trace spans. + +### Configuration + +```ini +# Recommended: native OTel metrics via OTLP/HTTP +[insight] +server=otel +endpoint=http://localhost:4318/v1/metrics +prefix=xrpld +``` + +Fallback (StatsD): + +```ini +[insight] +server=statsd +address=127.0.0.1:8125 +prefix=xrpld +``` + +### 2.1 Gauges + +| Prometheus Metric | Source File | Description | Typical Range | +| ------------------------------------------- | --------------------- | ----------------------------------------- | ------------------------------- | +| `ledgermaster_validated_ledger_age` | LedgerMaster.h | Seconds since last validated ledger | 0–10 (healthy), >30 (stale) | +| `ledgermaster_published_ledger_age` | LedgerMaster.h | Seconds since last published ledger | 0–10 (healthy) | +| `state_accounting_disconnected_duration` | NetworkOPs.cpp | Cumulative seconds in Disconnected state | Monotonic | +| `state_accounting_connected_duration` | NetworkOPs.cpp | Cumulative seconds in Connected state | Monotonic | +| `state_accounting_syncing_duration` | NetworkOPs.cpp | Cumulative seconds in Syncing state | Monotonic | +| `state_accounting_tracking_duration` | NetworkOPs.cpp | Cumulative seconds in Tracking state | Monotonic | +| `state_accounting_full_duration` | NetworkOPs.cpp | Cumulative seconds in Full state | Monotonic (should dominate) | +| `state_accounting_disconnected_transitions` | NetworkOPs.cpp | Count of transitions to Disconnected | Low | +| `state_accounting_connected_transitions` | NetworkOPs.cpp | Count of transitions to Connected | Low | +| `state_accounting_syncing_transitions` | NetworkOPs.cpp | Count of transitions to Syncing | Low | +| `state_accounting_tracking_transitions` | NetworkOPs.cpp | Count of transitions to Tracking | Low | +| `state_accounting_full_transitions` | NetworkOPs.cpp | Count of transitions to Full | Low (should be 1 after startup) | +| `peer_finder_active_inbound_peers` | PeerfinderManager.cpp | Active inbound peer connections | 0–85 | +| `peer_finder_active_outbound_peers` | PeerfinderManager.cpp | Active outbound peer connections | 10–21 | +| `overlay_peer_disconnects` | OverlayImpl.cpp | Cumulative peer disconnection count | Low growth | +| `overlay_peer_disconnects_charges` | OverlayImpl.cpp | Disconnects due to resource limit charges | Low growth (subset of above) | +| `jobq_job_count` | JobQueue.cpp | Current job queue depth (group `jobq`) | 0–100 (healthy) | + +**Grafana dashboard**: _Node Health_ (`node-health`) + +### 2.2 Counters + +| Prometheus Metric | Source File | Description | +| ------------------------- | ------------------ | --------------------------------------------- | +| `rpc_requests` | ServerHandler.cpp | Total RPC requests received | +| `ledger_fetches` | InboundLedgers.cpp | Inbound ledger fetch attempts | +| `ledger_history_mismatch` | LedgerHistory.cpp | Ledger hash mismatches detected | +| `warn` | Logic.h | Resource manager warnings issued | +| `drop` | Logic.h | Resource manager drops (connections rejected) | + +**Note**: With `server=otel`, `warn` and `drop` are properly exported as OTel Counter instruments. The previous StatsD `|m` type limitation no longer applies. + +**Grafana dashboard**: _RPC & Pathfinding_ (`rpc-pathfinding`) + +### 2.3 Histograms (Event timers) + +| Prometheus Metric | Source File | Unit | Description | +| ----------------- | ----------------- | ---- | ------------------------------ | +| `rpc_time` | ServerHandler.cpp | ms | RPC response time distribution | +| `rpc_size` | ServerHandler.cpp | ms\* | RPC response size (see note) | +| `ios_latency` | Application.cpp | ms | I/O service loop latency | +| `pathfind_fast` | PathRequests.h | ms | Fast pathfinding duration | +| `pathfind_full` | PathRequests.h | ms | Full pathfinding duration | + +Quantiles collected: 0th, 50th, 90th, 95th, 99th, 100th percentile. + +\* **`rpc_size` instrument mismatch (known issue):** response size in bytes is +recorded through the millisecond-scaled event histogram (`makeEvent`), so it is +exported as `rpc_size_milliseconds_bucket` with time-scaled boundaries that top +out at 5000. Byte values above ~5 KB saturate in the last bucket, so the +percentiles are not true byte sizes. The _RPC & Pathfinding_ panel is flagged +accordingly. A dedicated byte-unit histogram is needed to fix this; tracked +separately. + +**Grafana dashboards**: _Node Health_ (`ios_latency`), _RPC & Pathfinding_ (`rpc_time`, `rpc_size`, `pathfind_*`) + +### 2.4 Overlay Traffic Metrics + +For each of the 45+ overlay traffic categories (defined in `TrafficCount.h`), four gauges are emitted: + +- `{category}_bytes_in` +- `{category}_bytes_out` +- `{category}_messages_in` +- `{category}_messages_out` + +**Key categories**: + +| Category | Description | +| ----------------------------------------------------------------- | -------------------------- | +| `total` | All traffic aggregated | +| `overhead` / `overhead_overlay` | Protocol overhead | +| `transactions` / `transactions_duplicate` | Transaction relay | +| `proposals` / `proposals_untrusted` / `proposals_duplicate` | Consensus proposals | +| `validations` / `validations_untrusted` / `validations_duplicate` | Consensus validations | +| `ledger_data_get` / `ledger_data_share` | Ledger data exchange | +| `ledger_data_Transaction_Node_get/share` | Transaction node data | +| `ledger_data_Account_State_Node_get/share` | Account state node data | +| `ledger_data_Transaction_Set_candidate_get/share` | Transaction set candidates | +| `getObject` / `haveTxSet` / `ledgerData` | Object requests | +| `ping` / `status` | Keepalive and status | +| `set_get` | Set requests | + +**Grafana dashboards**: _Network Traffic_ (`network-traffic`), _Overlay Traffic Detail_ (`overlay-traffic-detail`), _Ledger Data & Sync_ (`ledger-data-sync`) + +### 2.5 Per-Job-Type Queue Gauges + +Three gauge families give per-job-type queue pressure. Before them the only +exported queue signal was the process-wide `jobq_job_count` +([§2.1](#21-gauges)), which cannot attribute pressure to a job type. + +| Prometheus Metric | Description | +| ------------------------- | --------------------------------------------------- | +| `jobq__waiting` | Backlog for this type: enqueued but not yet started | +| `jobq__running` | Currently executing for this type | +| `jobq__deferred` | Held back by this type's concurrency limit | + +The gauge members live on `JobTypeData` (`include/xrpl/core/JobTypeData.h`) and +are published by `JobQueue::collect()` +(`src/libxrpl/core/detail/JobQueue.cpp`), which reads the same +`waiting`/`running`/`deferred` counters under the mutex that guards them. Values +are clamped at zero before publication, because the gauge value type is unsigned +and an unclamped negative would wrap to ~1.8e19 and swamp every panel reading +the family. + +**Name derivation.** The collector is the `"jobq"` group +(`src/xrpld/app/main/Application.cpp`), so `GroupImp::makeName()` +(`src/libxrpl/beast/insight/Groups.cpp`) joins with a `.`, then +`OTelCollectorImp::formatName()` +(`src/libxrpl/beast/insight/OTelCollector.cpp`) lowercases and maps `.` to +`_`. The exported +name for `JtLedgerReq`, whose `JobTypeInfo` name is `ledgerRequest`, is +therefore `jobq_ledgerrequest_deferred` — bare and lowercase, with no `xrpld` +prefix. The same chain produces `jobq_job_count` from the gauge registered as +`job_count`. + +**Coverage.** Emitted for the **35** job types that are not special. `JobTypes` +defines 46 entries plus the `invalid` sentinel +(`include/xrpl/core/JobTypes.h`); `JobTypeInfo::special()` is `limit_ == 0`, and +11 of the 46 have `limit == 0`, so `JobTypeData`'s constructor creates gauges for +the remaining 35. A special type's gauge stays default-constructed, and a +default `beast::insight::Gauge` holds a null impl whose mutators are no-ops, so +assigning to it publishes nothing. + +**Why `deferred` is the metric to alert on.** `JobQueue::addJob()` never +rejects — it defers. A capped type under pressure therefore surfaces as latency +only after the fact, whereas a non-zero `deferred` reading precedes it. The +types where this bites are the ones with a low concurrency limit +(`JobTypes.h`): `JtPack` = 1 and `JtUpdatePf` = 1, `JtLedgerReq` = 3 and +`JtLedgerData` = 3, `JtTxnData` = 5. + +> **Sampling caveat.** These are sampled, not integrated. The values are read +> when the SDK's periodic reader invokes the observable callbacks, which run the +> collector hooks; the export interval is 1000 ms +> (`export_interval_millis` in `src/libxrpl/telemetry/Telemetry.cpp:441`) and +> hook invocation is debounced to at most once per 500 ms. A spike shorter than +> the interval can be missed entirely, so read these as pressure indicators +> rather than as exact peak depths. + +**Pipeline note.** Unlike the Phase 9 `job_*` counters and histograms, this +family flows through `beast::insight` → `OTelCollector`, **not** the +`XRPL_METRIC_*` macros. `JobQueue.cpp` is in `libxrpl` and those macros are +`xrpld`-only. The two are distinct pipelines — arrows **B** and **C** in the +[Data Flow Overview](#data-flow-overview) — each with its own `MeterProvider`, +reader, and OTLP/HTTP exporter, even though both request a meter named +`xrpld` / `1.0.0`. `OTelCollector` takes its meter from the **global** provider, +which `Telemetry` publishes and reads every 1000 ms; `MetricsRegistry` builds a +private provider it does not publish, read every 10000 ms +(`src/xrpld/telemetry/MetricsRegistry.cpp`). So `jobq__*` and +`job_*_total` reach Prometheus on different cadences and should not be assumed +sampled at the same instant. + +--- + +## 3. Grafana Dashboard Reference + +> **See also**: [05-configuration-reference.md](./05-configuration-reference.md) §5.8 for Grafana data source provisioning (Tempo, Prometheus) and TraceQL query examples. + +### 3.1 Span-Derived Dashboards (5) + +| Dashboard | UID | Data Source | Key Panels | +| -------------------- | ---------------------- | ------------------------ | ---------------------------------------------------------------------------------- | +| RPC Performance | `rpc-performance` | Prometheus (SpanMetrics) | Request rate by command, p95 latency by command, error rate, heatmap, top commands | +| Transaction Overview | `transaction-overview` | Prometheus (SpanMetrics) | Processing rate, latency p95/p50, local vs relay split, apply duration, heatmap | +| Consensus Health | `consensus-health` | Prometheus (SpanMetrics) | Round duration p95/p50, proposals rate, close duration, mode timeline, heatmap | +| Ledger Operations | `ledger-operations` | Prometheus (SpanMetrics) | Build rate, build duration, validation rate, store rate, build vs close comparison | +| Peer Network | `peer-network` | Prometheus (SpanMetrics) | Proposal receive rate, validation receive rate, trusted vs untrusted breakdown | + +### 3.2 System Metrics Dashboards (5) + +| Dashboard | UID | Data Source | Key Panels | +| ---------------------- | ------------------------ | ----------------- | --------------------------------------------------------------------------------- | +| Node Health | `node-health` | Prometheus (OTLP) | Ledger age, operating mode, I/O latency, job queue, fetch rate | +| Network Traffic | `network-traffic` | Prometheus (OTLP) | Active peers, disconnects, bytes in/out, messages in/out, traffic by category | +| RPC & Pathfinding | `rpc-pathfinding` | Prometheus (OTLP) | RPC rate, response time/size, pathfinding duration, resource warnings/drops | +| Overlay Traffic Detail | `overlay-traffic-detail` | Prometheus (OTLP) | Squelch, overhead, validator lists, set get/share, have/requested tx, proof paths | +| Ledger Data & Sync | `ledger-data-sync` | Prometheus (OTLP) | Ledger data exchange, legacy ledger share/get, getobject by type, traffic heatmap | + +### 3.3 Deployment-Tier Template Variables + +Every dashboard carries seven filtering template variables (each variable name +matches its Prometheus label), letting one Grafana stack be sliced by tier and +by perf-comparison run: + +| Variable | Source label | Description | +| ------------------------- | ------------------------ | ---------------------------------------------------------------- | +| `$node` | `service_instance_id` | Filter by xrpld node instance | +| `$service_name` | `service_name` | Filter by service (`service.name`, e.g. `xrpld`) | +| `$deployment_environment` | `deployment_environment` | Filter by deployment tier (`local` / `test` / `ci` / `prod`) | +| `$xrpl_network_type` | `xrpl_network_type` | Filter by network (`mainnet` / `testnet` / `devnet` / `perf`) | +| `$xrpl_work_item` | `xrpl_work_item` | Filter by perf-iac work item / ticket (e.g. `RIPD-7455`) | +| `$xrpl_branch` | `xrpl_branch` | Filter by comparison side (`baseline::` / `test:…`) | +| `$xrpl_node_role` | `xrpl_node_role` | Filter by node role (`validator` / `peer`) | + +The last three are populated only during perf-iac comparison runs (stamped as +resource attributes by perf-iac's own alloy pipeline, not the repo collector). +Outside those runs the labels are absent; the filters default to **All**, which +matches series lacking the label so every dashboard still renders. + +See [telemetry-runbook.md](../docs/telemetry-runbook.md) "Deployment Tiers" +for how the tier attributes are set and reach metrics. + +### 3.4 Accessing the Dashboards + +1. Open Grafana at **http://localhost:3000** +2. Navigate to **Dashboards → xrpld** folder +3. All 10 dashboards are auto-provisioned from `docker/telemetry/grafana/dashboards/` + +--- + +## 4. Tempo Trace Search Guide + +> **See also**: [08-appendix.md](./08-appendix.md) §8.2 for span hierarchy visualizations. [05-configuration-reference.md](./05-configuration-reference.md) §5.8.5 for TraceQL query examples. + +### Finding Traces by Type + +| What to Find | Tempo TraceQL Query | +| ------------------------ | ------------------------------------------------------------------------------ | +| All RPC calls | `{resource.service.name="xrpld" && name="rpc.http_request"}` | +| Specific RPC command | `{resource.service.name="xrpld" && name="rpc.command.server_info"}` | +| Slow RPC calls | `{resource.service.name="xrpld" && name=~"rpc.command.*"} \| duration > 100ms` | +| Failed RPC calls | `{span.rpc_status="error"}` | +| gRPC method calls | `{resource.service.name="xrpld" && name="grpc.GetLedger"}` | +| Specific transaction | `{span.tx_hash=""}` | +| Local transactions only | `{span.local=true}` | +| Consensus rounds | `{resource.service.name="xrpld" && name="consensus.round"}` | +| Rounds by mode | `{span.consensus_mode="Proposing"}` | +| Specific ledger | `{span.ledger_seq=12345}` | +| Peer proposals (trusted) | `{span.proposal_trusted=true}` | + +### Trace Structure + +A typical RPC trace shows the span hierarchy: + +``` +rpc.http_request (ServerHandler) + └── rpc.process (ServerHandler) + └── rpc.command.server_info (RPCHandler) +``` + +A consensus round groups its lifecycle spans under a single root +(`consensus.round`); the build/ledger spans run as their own trees: + +``` +consensus.round (root — one per round) + ├── consensus.phase.open (open phase) + ├── consensus.proposal.send (broadcast proposal) + ├── consensus.ledger_close (close event) + ├── consensus.establish (establish phase) + ├── consensus.update_positions (position updates) + ├── consensus.check (threshold check) + ├── consensus.accept (accept result) + │ └── consensus.accept.apply (apply, jtACCEPT thread) + └── consensus.validation.send (send validation, follows-from link) + +ledger.build (build new ledger) + └── tx.apply (apply transaction set) +ledger.validate (promote to validated) +ledger.store (persist to DB) +``` + +--- + +## 5. Prometheus Query Examples + +> **See also**: [05-configuration-reference.md](./05-configuration-reference.md) §5.8.7 for correlating Prometheus system metrics with trace-derived metrics. + +### Span-Derived Metrics + +```promql +# RPC request rate by command (last 5 minutes) +sum by (command) (rate(span_calls_total{span_name=~"rpc.command.*"}[5m])) + +# RPC p95 latency by command +histogram_quantile(0.95, sum by (le, command) (rate(span_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m]))) + +# Consensus round duration p95 +histogram_quantile(0.95, sum by (le) (rate(span_duration_milliseconds_bucket{span_name="consensus.round"}[5m]))) + +# Transaction processing rate (local vs relay) +sum by (local) (rate(span_calls_total{span_name="tx.process"}[5m])) + +# Trusted vs untrusted proposal rate +sum by (proposal_trusted) (rate(span_calls_total{span_name="peer.proposal.receive"}[5m])) +``` + +### StatsD Metrics + +```promql +# Validated ledger age (should be < 10s) +ledgermaster_validated_ledger_age + +# Active peer count +peer_finder_active_inbound_peers + peer_finder_active_outbound_peers + +# RPC response time p95 +histogram_quantile(0.95, rpc_time_bucket) + +# Total network bytes in (rate) +rate(total_bytes_in[5m]) + +# Operating mode (should be "Full" after startup) +state_accounting_full_duration +``` + +--- + +## 5a. Log-Trace Correlation (Phase 8) + +> **Plan details**: [06-implementation-phases.md §6.8.1](./06-implementation-phases.md) — motivation, architecture, Mermaid diagrams +> **Task breakdown**: [Phase8_taskList.md](./Phase8_taskList.md) — per-task implementation details + +Phase 8 injects OTel trace context into xrpld's `Logs::format()` output, enabling log-trace correlation. When a log line is emitted within an active OTel span, the trace and span identifiers are automatically appended after the severity field: + +### Log Format + +``` + : trace_id=<32hex> span_id=<16hex> +``` + +Example: + +``` +2024-Jan-15 10:30:45.123456 UTC LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42 +``` + +- **`trace_id=`** — 32-character lowercase hex trace identifier. Links to the distributed trace in Tempo. +- **`span_id=`** — 16-character lowercase hex span identifier. Identifies the specific span within the trace. +- **Only present** when the log is emitted within an active OTel span. Log lines outside of traced code paths have no trace context fields. + +### Implementation + +The trace context injection is implemented in `Logs::format()` (`src/libxrpl/basics/Log.cpp`), guarded by `#ifdef XRPL_ENABLE_TELEMETRY`. It checks the thread-local runtime context value directly (via `RuntimeContext::GetCurrent().GetValue(kSpanKey)`) to avoid the heap allocation that `GetSpan()` performs on the no-span path. On threads without an active span, the cost is a thread-local read + variant type check (~15-20ns). On the active-span path, total cost is ~50ns per log call. + +### Log Ingestion Pipeline + +``` +xrpld debug.log -> OTel Collector filelog receiver -> regex_parser -> Loki exporter -> Grafana Loki +``` + +The OTel Collector's `filelog` receiver tails `debug.log` files and uses a `regex_parser` operator to extract structured fields: + +| Field | Type | Description | +| ----------- | -------- | -------------------------------------------------------- | +| `timestamp` | datetime | Log timestamp | +| `partition` | string | Log partition (e.g., `LedgerMaster`, `PeerImp`) | +| `severity` | string | Severity code (`TRC`, `DBG`, `NFO`, `WRN`, `ERR`, `FTL`) | +| `trace_id` | string | 32-hex trace identifier (optional) | +| `span_id` | string | 16-hex span identifier (optional) | +| `message` | string | Log message body | + +### Grafana Correlation + +Bidirectional linking between logs and traces is configured via Grafana datasource provisioning: + +- **Tempo -> Loki** (`tracesToLogs`): Clicking "Logs for this trace" on a Tempo trace view filters Loki logs by `trace_id`, showing all log lines from that trace. +- **Loki -> Tempo** (`derivedFields`): A regex-based derived field on the Loki datasource extracts `trace_id` from log lines and renders it as a clickable link to the corresponding trace in Tempo. + +### Loki Backend + +Grafana Loki (v3.4.2) serves as the log storage backend. It receives log entries from the OTel Collector's `otlphttp/loki` exporter via the native OTLP endpoint at `http://loki:3100/otlp`. + +### LogQL Query Examples + +```logql +# Find all logs for a specific trace +{job="xrpld"} |= "trace_id=abc123def456789012345678abcdef01" + +# Error logs with trace context +{job="xrpld"} |= "ERR" |= "trace_id=" + +# Logs from a specific partition with trace context +{job="xrpld"} |= "LedgerMaster" | regexp `trace_id=(?P[a-f0-9]+)` | trace_id != "" + +# Count traced log lines over time +count_over_time({job="xrpld"} |= "trace_id=" [5m]) +``` + +--- + +## 5b. Internal Metric Gap Fill (Phase 9) + +> **Status**: Implemented. +> **Plan details**: [06-implementation-phases.md §6.8.2](./06-implementation-phases.md) — motivation, architecture, third-party context +> **Task breakdown**: [Phase9_taskList.md](./Phase9_taskList.md) — per-task implementation details + +Phase 9 fills the metrics that exist inside xrpld but previously lacked time-series export. It +uses a hybrid approach: `beast::insight` extensions for NodeStore I/O plus OTel `ObservableGauge` +async callbacks for new categories. + +> **Authoritative metric names live in [§ Phase 9: OTel SDK-Exported Metrics](#phase-9-otel-sdk-exported-metrics-metricsregistry) below.** +> Most internal metrics are emitted as **labeled** gauges — one instrument carrying many logical +> values via a `metric` label (e.g. `cache_metrics{metric="sle_hit_rate"}`, +> `txq_metrics{metric="txq_count"}`, `load_factor_metrics{metric="load_factor"}`, +> `nodestore_state{metric="node_reads_total"}`) — not the flat per-name form. Query the +> labeled names; the flat names (`cache_sle_hit_rate`, `txq_count`, …) are **not** emitted. + +#### Server Info (via OTel MetricsRegistry) + +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------------- | ----- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `server_info{metric="server_state"}` | Gauge | `metric` | Operating mode (0=DISCONNECTED .. 4=FULL) | +| `server_info{metric="uptime"}` | Gauge | `metric` | Seconds since server start | +| `server_info{metric="peers"}` | Gauge | `metric` | Total connected peers | +| `server_info{metric="validated_ledger_seq"}` | Gauge | `metric` | Validated ledger sequence number | +| `server_info{metric="ledger_current_index"}` | Gauge | `metric` | Current open ledger sequence | +| `server_info{metric="peer_disconnects_resources"}` | Gauge | `metric` | Cumulative resource-related peer disconnects | +| `server_info{metric="last_close_proposers"}` | Gauge | `metric` | Proposers in last closed round | +| `server_info{metric="last_close_converge_time_ms"}` | Gauge | `metric` | Last close convergence time (milliseconds) | +| `server_info{metric="last_close_time"}` | Gauge | `metric` | Network close time of last closed ledger (NetClock secs since XRPL epoch). Query `time() - (value + 946684800)` for last-close age (staleness). Use `1/rate(ledgers_closed_total)` — not a gauge delta — for the close interval | + +#### Build Info (via OTel MetricsRegistry) + +| Prometheus Metric | Type | Labels | Description | +| ----------------------------- | ----- | --------- | --------------------------------- | +| `build_info{version=""}` | Gauge | `version` | Info-style metric, always value 1 | + +#### Complete Ledger Ranges (via OTel MetricsRegistry) + +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------- | ----- | --------------- | --------------------------- | +| `complete_ledgers{bound="start",index=""}` | Gauge | `bound`,`index` | Start of contiguous range N | +| `complete_ledgers{bound="end",index=""}` | Gauge | `bound`,`index` | End of contiguous range N | + +#### Database Metrics (via OTel MetricsRegistry) + +| Prometheus Metric | Type | Labels | Description | +| ------------------------------------------- | ----- | -------- | --------------------------------- | +| `db_metrics{metric="db_kb_total"}` | Gauge | `metric` | Total database size (KB) | +| `db_metrics{metric="db_kb_ledger"}` | Gauge | `metric` | Ledger database size (KB) | +| `db_metrics{metric="db_kb_transaction"}` | Gauge | `metric` | Transaction database size (KB) | +| `db_metrics{metric="historical_perminute"}` | Gauge | `metric` | Historical ledger fetches per min | + +#### Extended Cache Metrics (additions to existing cache_metrics) + +| Prometheus Metric | Type | Labels | Description | +| --------------------------------- | ----- | -------- | ------------------------- | +| `cache_metrics{metric="al_size"}` | Gauge | `metric` | AcceptedLedger cache size | + +#### Extended NodeStore Metrics (additions to existing nodestore_state) + +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------------------- | ----- | -------- | ----------------------------------- | +| `nodestore_state{metric="node_reads_duration_us"}` | Gauge | `metric` | Cumulative read time (microseconds) | +| `nodestore_state{metric="read_request_bundle"}` | Gauge | `metric` | Read request bundle count | +| `nodestore_state{metric="read_threads_running"}` | Gauge | `metric` | Active read threads | +| `nodestore_state{metric="read_threads_total"}` | Gauge | `metric` | Total read threads configured | + +#### Job Queue and GetObject Additions + +Three further additions are catalogued with the families they extend rather than +repeated here: + +- A `handler` label on the five `job_*` instruments, so producers sharing one + job type stay individually attributable — see + [Per-Job-Type Metrics](#per-job-type-metrics-synchronous-countershistogram). +- Five `getobject_*` instruments covering the `TMGetObjectByHash` request path — + see [GetObject Request Path](#getobject-request-path-synchronous-countershistograms). +- Three per-job-type queue gauge families (`jobq__waiting` / + `_running` / `_deferred`). These travel the `beast::insight` pipeline, not the + OTel SDK one, so they are documented in + [§2.5](#25-per-job-type-queue-gauges). + +### New Grafana Dashboards (Phase 9) + +| Dashboard | UID | Data Source | Key Panels | +| ------------------ | ------------ | ----------- | ----------------------------------------------------------------- | +| Fee Market & TxQ | `fee-market` | Prometheus | TxQ depth/capacity, fee levels, load factor breakdown, escalation | +| Job Queue Analysis | `job-queue` | Prometheus | Per-job rates, queue wait times, execution times, queue depth | + +--- + +## 5c. Future: Synthetic Workload Generation & Telemetry Validation (Phase 10) + +> **Plan details**: [06-implementation-phases.md §6.8.3](./06-implementation-phases.md) — motivation, architecture +> **Task breakdown**: [Phase10_taskList.md](./Phase10_taskList.md) — per-task implementation details +> **Tools**: [docker/telemetry/workload/](../docker/telemetry/workload/) — RPC load generator, transaction submitter, validation suite, benchmarks + +Phase 10 builds a 5-node validator docker-compose harness with RPC load generators, transaction submitters, and automated validation scripts that verify all spans, metrics, dashboards, and log-trace correlation work end-to-end. Includes a benchmark suite comparing telemetry-ON vs telemetry-OFF overhead. + +### Running the Validation Suite + +```bash +# Full end-to-end validation (start cluster, generate load, validate): +docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld + +# Validation only (assumes stack and cluster are already running): +python3 docker/telemetry/workload/validate_telemetry.py --report /tmp/report.json + +# Performance benchmark (baseline vs telemetry): +docker/telemetry/workload/benchmark.sh --xrpld .build/xrpld --duration 300 +``` + +### Validated Telemetry Inventory + +> **Counting note — families vs series.** A _metric family_ is one distinct Prometheus `__name__` +> (histogram `_bucket`/`_count`/`_sum` collapsed to one). A _series_ is a family × its label +> combinations. The legacy overlay-traffic block is the bulk of the count: ~56 message categories × +> 4 (`_bytes_in/_out`, `_messages_in/_out`) ≈ 224 families on its own. The labeled gauges +> (`cache_metrics{metric}`, …) are few families but many series. Validate against the figures +> below as **families currently emitting** (idle nodes under-report — workload-gated metrics such as +> per-RPC/error counters appear only once exercised, which is Phase 10's purpose). + +| Category | Expected Count | Validation Method | Config File | +| ------------------------------ | ------------------------- | -------------------------------- | ----------------------- | +| Trace spans | ~37 (required + optional) | Tempo API query | `expected_spans.json` | +| Span attributes | per-span assertion | Per-span attribute assertion | `expected_spans.json` | +| Legacy beast::insight families | ~270 (≈224 traffic) | Prometheus `__name__` query | `expected_metrics.json` | +| Native MetricsRegistry | 35 instruments | Prometheus query | `expected_metrics.json` | +| Call-site `XRPL_METRIC_*` | 7 instruments | Prometheus query | `expected_metrics.json` | +| Per-job-type gauges | 105 (35 types × 3) | Prometheus `__name__` query | `expected_metrics.json` | +| SpanMetrics RED | 4 per span | Prometheus query | `expected_metrics.json` | +| Grafana dashboards | 15 | Dashboard API "no data" check | `expected_metrics.json` | +| Log-trace links | Present | Loki query + Tempo reverse check | — | + +The two added rows are the families that do not originate as `MetricsRegistry` +members. **Call-site** instruments are declared by the `XRPL_METRIC_*` macros +(7 distinct names: `rpc_in_flight_requests`, `ledgers_closed_total`, and the +five `getobject_*`). Two of those seven are workload-gated in a way that makes a +zero reading uninformative: `getobject_rejected_total` needs a non-conforming +request, and the `getobject_*` family as a whole needs an inbound +`TMGetObjectByHash`. **Per-job-type gauges** are the `beast::insight` families +from [§2.5](#25-per-job-type-queue-gauges); all 105 should be present on any +running node, but `_deferred` reads zero unless a capped type is actually +saturated. + +### Performance Overhead Targets + +| Metric | Target | Measurement Method | +| ----------------- | ------------ | ----------------------------------- | +| CPU overhead | < 3% | ps avg CPU% baseline vs telemetry | +| Memory overhead | < 5MB | ps peak RSS baseline vs telemetry | +| RPC p99 latency | < 2ms impact | server_info round-trip timing | +| Throughput impact | < 5% | Ledger close rate comparison | +| Consensus impact | < 1% | Consensus round time p95 comparison | + +--- + +## 5d. Future: Third-Party Data Collection Pipelines (Phase 11) + +> **Status**: Planned, not yet implemented. +> **Plan details**: [06-implementation-phases.md §6.8.4](./06-implementation-phases.md) — motivation, architecture, consumer gap analysis +> **Task breakdown**: [Phase11_taskList.md](./Phase11_taskList.md) — per-task implementation details + +Phase 11 builds a custom OTel Collector receiver (Go) that polls xrpld's admin RPCs and exports `xrpl_*` metrics for external consumers. No xrpld code changes. + +### Exported Metrics (via Custom OTel Collector Receiver) + +#### Node Health (from server_info) + +| Prometheus Metric | Type | Description | +| --------------------------------------- | ----- | ----------------------------------------------- | +| `xrpl_server_state` | Gauge | Operating mode (0=disconnected ... 5=proposing) | +| `xrpl_server_state_duration_seconds` | Gauge | Seconds in current state | +| `xrpl_uptime_seconds` | Gauge | Consecutive seconds running | +| `xrpl_io_latency_ms` | Gauge | I/O subsystem latency | +| `xrpl_amendment_blocked` | Gauge | 1 if amendment-blocked, 0 otherwise | +| `xrpl_peers_count` | Gauge | Connected peers | +| `xrpl_validated_ledger_seq` | Gauge | Latest validated ledger sequence | +| `xrpl_validated_ledger_age_seconds` | Gauge | Seconds since last validated close | +| `xrpl_last_close_proposers` | Gauge | Proposers in last consensus round | +| `xrpl_last_close_converge_time_seconds` | Gauge | Last consensus round duration | +| `xrpl_load_factor` | Gauge | Transaction cost multiplier | +| `xrpl_state_duration_seconds` | Gauge | Per-state duration (`state` label) | +| `xrpl_state_transitions_total` | Gauge | Per-state transition count (`state` label) | + +#### Peer Topology (from peers) + +| Prometheus Metric | Type | Description | +| --------------------------- | ----- | ----------------------------------- | +| `xrpl_peers_inbound_count` | Gauge | Inbound peer connections | +| `xrpl_peers_outbound_count` | Gauge | Outbound peer connections | +| `xrpl_peer_latency_p50_ms` | Gauge | Median peer latency | +| `xrpl_peer_latency_p95_ms` | Gauge | p95 peer latency | +| `xrpl_peer_version_count` | Gauge | Peers per version (`version` label) | +| `xrpl_peer_diverged_count` | Gauge | Peers with diverged tracking status | + +#### Validator & Amendment (from validators, feature) + +| Prometheus Metric | Type | Description | +| ------------------------------------- | ----- | --------------------------------------- | +| `xrpl_trusted_validators_count` | Gauge | UNL validator count | +| `xrpl_amendment_enabled_count` | Gauge | Enabled amendments | +| `xrpl_amendment_majority_count` | Gauge | Amendments with majority | +| `xrpl_amendment_unsupported_majority` | Gauge | 1 if unsupported amendment has majority | +| `xrpl_validator_list_active` | Gauge | 1 if validator list is active | + +#### Fee Market (from fee) + +| Prometheus Metric | Type | Description | +| -------------------------------- | ----- | ------------------------------------- | +| `xrpl_fee_open_ledger_fee_drops` | Gauge | Minimum fee for open ledger inclusion | +| `xrpl_fee_median_fee_drops` | Gauge | Median fee level | +| `xrpl_fee_queue_size` | Gauge | Current transaction queue depth | +| `xrpl_fee_current_ledger_size` | Gauge | Transactions in current open ledger | + +#### DEX & AMM (optional, from book_offers, amm_info) + +| Prometheus Metric | Type | Labels | Description | +| -------------------------- | ----- | --------------------- | ---------------------- | +| `xrpl_amm_tvl_drops` | Gauge | `pool=""` | Total value locked | +| `xrpl_amm_trading_fee` | Gauge | `pool=""` | Pool trading fee (bps) | +| `xrpl_orderbook_bid_depth` | Gauge | `pair=""` | Total bid volume | +| `xrpl_orderbook_ask_depth` | Gauge | `pair=""` | Total ask volume | +| `xrpl_orderbook_spread` | Gauge | `pair=""` | Best bid-ask spread | + +### Phase 9: OTel SDK-Exported Metrics (MetricsRegistry) + +Phase 9 introduces the `MetricsRegistry` class (`src/xrpld/telemetry/MetricsRegistry.h/.cpp`) +which registers metrics directly with the OpenTelemetry Metrics SDK. These are exported +via OTLP/HTTP to the OTel Collector and scraped by Prometheus. + +#### NodeStore I/O (Observable Gauge — `nodestore_state`) + +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------- | ----- | -------- | ------------------------------------ | +| `nodestore_state{metric="node_reads_total"}` | Gauge | `metric` | Cumulative NodeStore read operations | +| `nodestore_state{metric="node_reads_hit"}` | Gauge | `metric` | Reads served from cache | +| `nodestore_state{metric="node_writes"}` | Gauge | `metric` | Cumulative write operations | +| `nodestore_state{metric="node_written_bytes"}` | Gauge | `metric` | Cumulative bytes written | +| `nodestore_state{metric="node_read_bytes"}` | Gauge | `metric` | Cumulative bytes read | +| `nodestore_state{metric="write_load"}` | Gauge | `metric` | Current write load score | +| `nodestore_state{metric="read_queue"}` | Gauge | `metric` | Items in read prefetch queue | + +#### Cache Hit Rates & Sizes (Observable Gauge — `cache_metrics`) + +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------- | ----- | -------- | ----------------------------- | +| `cache_metrics{metric="sle_hit_rate"}` | Gauge | `metric` | SLE cache hit rate (0.0-1.0) | +| `cache_metrics{metric="ledger_hit_rate"}` | Gauge | `metric` | Ledger cache hit rate | +| `cache_metrics{metric="al_hit_rate"}` | Gauge | `metric` | AcceptedLedger cache hit rate | +| `cache_metrics{metric="treenode_cache_size"}` | Gauge | `metric` | SHAMap TreeNode cache entries | +| `cache_metrics{metric="treenode_track_size"}` | Gauge | `metric` | Tracked tree nodes | +| `cache_metrics{metric="fullbelow_size"}` | Gauge | `metric` | FullBelow cache entries | + +#### Transaction Queue (Observable Gauge — `txq_metrics`) + +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------------- | ----- | -------- | -------------------------------- | +| `txq_metrics{metric="txq_count"}` | Gauge | `metric` | Transactions currently in queue | +| `txq_metrics{metric="txq_max_size"}` | Gauge | `metric` | Maximum queue capacity | +| `txq_metrics{metric="txq_in_ledger"}` | Gauge | `metric` | Transactions in open ledger | +| `txq_metrics{metric="txq_per_ledger"}` | Gauge | `metric` | Expected transactions per ledger | +| `txq_metrics{metric="txq_reference_fee_level"}` | Gauge | `metric` | Reference fee level | +| `txq_metrics{metric="txq_min_processing_fee_level"}` | Gauge | `metric` | Minimum fee to get processed | +| `txq_metrics{metric="txq_med_fee_level"}` | Gauge | `metric` | Median fee level in queue | +| `txq_metrics{metric="txq_open_ledger_fee_level"}` | Gauge | `metric` | Open ledger fee escalation level | + +#### Per-RPC Method Metrics (Synchronous Counters/Histogram) + +| Prometheus Metric | Type | Labels | Description | +| --------------------------- | ------------- | ----------------- | ------------------------------------------------------ | +| `rpc_method_started_total` | Counter | `method=""` | RPC calls started | +| `rpc_method_finished_total` | Counter | `method=""` | RPC calls completed successfully | +| `rpc_method_errored_total` | Counter | `method=""` | RPC calls that errored | +| `rpc_method_us` | Histogram | `method=""` | Execution time distribution (us) | +| `rpc_in_flight_requests` | UpDownCounter | (none) | RPC calls currently executing (+1 rpcStart, -1 rpcEnd) | + +`rpc_in_flight_requests` is emitted at its call site via the `XRPL_METRIC_UPDOWN_ADD` +macro (see `src/xrpld/telemetry/MetricMacros.h` and `PerfLogImp.cpp`), not through a +`MetricsRegistry` member. As an UpDownCounter it carries no `_total` suffix (that is +reserved for monotonic counters). + +#### Per-Job-Type Metrics (Synchronous Counters/Histogram) + +| Prometheus Metric | Type | Labels | Description | +| -------------------- | --------- | --------------------------------------- | --------------------------------- | +| `job_queued_total` | Counter | `job_type=""`, `handler=""` | Jobs enqueued | +| `job_started_total` | Counter | `job_type=""`, `handler=""` | Jobs started | +| `job_finished_total` | Counter | `job_type=""`, `handler=""` | Jobs completed | +| `job_queued_us` | Histogram | `job_type=""`, `handler=""` | Queue wait time distribution (us) | +| `job_running_us` | Histogram | `job_type=""`, `handler=""` | Execution time distribution (us) | + +All five are recorded from `PerfLogImp` (`jobQueue()`, `jobStart()`, +`jobFinish()`) through `MetricsRegistry::recordJobQueued/Started/Finished`. +A counter and its paired histogram always carry the identical label set, so +the two can be joined in a query. + +**The `handler` label.** `job_type` alone cannot attribute load to a producer, +because several producers share one job type: `RcvGetLedger` and +`RcvGetObjByHash` both run as `JtLedgerReq`, and before this label they were +indistinguishable. `handler` is the `addJob` name, so each producer gets its +own series. + +The name is not used raw. `MetricsRegistry::sanitiseHandler()` keeps it only +when it is non-empty **and** every character is an ASCII letter; anything else +becomes the constant `MetricsRegistry::kHandlerOther`, `"other"`. The rule +exists because two job names embed a ledger sequence — `"Pub" + seq` in +`LedgerPersistence.cpp` and `"OB" + seq` in `OrderBookDBImpl.cpp` — which raw +would mint a fresh series per ledger. Both always contain digits, so both +always fold to `other` by construction. The label domain is therefore a +function of the string literals in the source and cannot grow at runtime; +a name added later that fails the rule degrades to `other` rather than +becoming unbounded. + +Current cardinality: **44 values** — 43 names pass through unchanged, plus +`other`. Five production names fail the letters-only rule and fold into +`other`: + +| Job Name | Job Type | Why it folds | +| ------------- | ------------------- | ------------ | +| `GetConsL1` | `JtAdvance` | digits | +| `GetConsL2` | `JtAdvance` | digits | +| `gRPC-Client` | `JtRpc` | hyphen | +| `RPC-Client` | `JtClientRpc` | hyphen | +| `WS-Client` | `JtClientWebsocket` | hyphen | + +> **`handler="other"` is a mixed bucket, not one producer.** It aggregates the +> five names above plus both dynamic names, so a rate or quantile on it is a +> sum across unrelated work. `GetConsL1` and `GetConsL2` are the sharpest case: +> they are two distinct `JtAdvance` producers that land in the same bucket and +> are mutually inseparable. Filter by `job_type` alongside `handler` to narrow +> it, and read `handler="other"` series as an aggregate only. + +#### GetObject Request Path (Synchronous Counters/Histograms) + +Instruments for the `TMGetObjectByHash` peer request path. Names, label keys, +and label values are the `constexpr` constants in +`include/xrpl/telemetry/GetObjectMetricNames.h`. All five are declared at their +call sites in `src/xrpld/overlay/detail/PeerImp.cpp` via the `XRPL_METRIC_*` +macros, not as `MetricsRegistry` members: the two rejection counters in +`onMessage(TMGetObjectByHash)`, the other three in the +`recordGetObjectMetrics()` helper. + +| Prometheus Metric | Type | Labels | Description | +| --------------------------- | --------- | ----------------------------------------------- | ------------------------------------------------------------ | +| `getobject_lookup_us` | Histogram | (none) | Wall time in the NodeStore fetch loop (us), once per request | +| `getobject_request_objects` | Histogram | (none) | Objects requested per message | +| `getobject_lookups_total` | Counter | `result="hit"` \| `"miss"` | NodeStore lookups, added once per request with batch totals | +| `getobject_rejected_total` | Counter | `reason="oversize"` \| `"malformed_ledgerhash"` | Requests refused before any NodeStore access | +| `getobject_charge` | Histogram | (none) | Dynamic component of the differential resource charge | + +**Per request, not per object.** `getobject_lookup_us` times the whole fetch +loop once, and `getobject_lookups_total` adds the batch hit and miss totals in +two calls. Incrementing per object on a loop bounded by +`Tuning::kHardMaxReplyNodes` (12288) would cost measurably and add no +information the batch totals do not already carry. + +**All three histograms need an explicit bucket view.** The SDK's default +histogram boundaries top out at 10000. Every one of these three exceeds that, so +without a view their top quantiles would all read as a flat 10000. Six views are +registered in `src/xrpld/telemetry/MetricsRegistry.cpp`, and three of the six are +for this family: + +| Instrument | View helper | Boundaries | +| --------------------------- | ------------------------------- | ------------------------------------------------------ | +| `getobject_lookup_us` | `addMicrosecondHistogramView()` | The shared µs ladder, 100 µs to 60 s (16 buckets) | +| `getobject_request_objects` | `addHistogramView()`, own set | `1, 2, 4, 8, 16, 64, 256, 1024, 4096, 12288` | +| `getobject_charge` | `addHistogramView()`, own set | `0, 100, 500, 1000, 5000, 10000, 25000, 50000, 100000` | + +The other three views are `addMicrosecondHistogramView()` on `job_queued_us`, +`job_running_us`, and `rpc_method_us` — four µs-ladder views plus these two +custom sets. + +**Why the latter two do not use the µs ladder.** They are not durations. The µs +ladder's buckets are chosen for time (sub-millisecond jobs through multi-second +stalls), so applying it to a count or a charge would place almost every +observation in one or two buckets and make the distribution unreadable. +`addHistogramView()` exists to take caller-supplied boundaries for exactly this +case. + +- **Counts** run 1 to `Tuning::kHardMaxReplyNodes` (12288). The low end is + fine-grained because the honest sync path asks for at most 8 objects, so the + interesting distinction is between a normal request and a large one. The upper + bounds follow the charge size bands — `kBandSmallMax` (64) and + `kBandMediumMax` (1024) — up to the hard cap, so a bucket boundary coincides + with each price change. +- **Charges** run 0 to roughly 99k for a full-size all-miss request. Two of the + boundaries are the resource thresholds that decide a peer's fate: + `Resource::kWarningThreshold` (5000) and `Resource::kDropThreshold` (25000), + both in `include/xrpl/resource/detail/Tuning.h`. Placing bucket edges exactly + there lets a panel read off how close real charges run to a warning or a drop, + rather than interpolating across an edge. + +`getobject_lookup_us` is named by a shared constant rather than a literal because +it is referenced from both the record site and the view registration; a drifted +spelling would silently drop the override. + +> **A zero `getobject_rejected_total` does not prove the counter works.** Both +> gates it counts (`reason="oversize"`, `reason="malformed_ledgerhash"`) fire +> only on non-conforming requests, so on a healthy network the expected reading +> is zero. Validate it with a deliberately malformed request, not by looking for +> a series. + +#### Counted Object Instances (Observable Gauge — `object_count`) + +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------- | ----- | --------------- | ------------------------------ | +| `object_count{type="transaction"}` | Gauge | `type=""` | Live Transaction objects | +| `object_count{type="ledger"}` | Gauge | `type=""` | Live Ledger objects | +| `object_count{type="nodeobject"}` | Gauge | `type=""` | Live NodeObject instances | +| `object_count{type="sttx"}` | Gauge | `type=""` | Serialized transaction objects | +| `object_count{type="stledgerentry"}` | Gauge | `type=""` | Serialized ledger entries | +| `object_count{type="inboundledger"}` | Gauge | `type=""` | Ledgers being fetched | +| `object_count{type="pathfinder"}` | Gauge | `type=""` | Active pathfinding operations | +| `object_count{type="pathrequest"}` | Gauge | `type=""` | Active path requests | +| `object_count{type="hashrouterentry"}` | Gauge | `type=""` | Hash router entries | + +#### Load Factor Breakdown (Observable Gauge — `load_factor_metrics`) + +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------------------- | ----- | -------- | --------------------------------------- | +| `load_factor_metrics{metric="load_factor"}` | Gauge | `metric` | Combined transaction cost multiplier | +| `load_factor_metrics{metric="load_factor_server"}` | Gauge | `metric` | Server + cluster + network contribution | +| `load_factor_metrics{metric="load_factor_local"}` | Gauge | `metric` | Local server load only | +| `load_factor_metrics{metric="load_factor_net"}` | Gauge | `metric` | Network-wide load estimate | +| `load_factor_metrics{metric="load_factor_cluster"}` | Gauge | `metric` | Cluster peer load | +| `load_factor_metrics{metric="load_factor_fee_escalation"}` | Gauge | `metric` | Open ledger fee escalation | +| `load_factor_metrics{metric="load_factor_fee_queue"}` | Gauge | `metric` | Queue entry fee level | + +#### Prometheus Query Examples (Phase 9) + +```promql +# NodeStore cache hit ratio +nodestore_state{metric="node_reads_hit"} / nodestore_state{metric="node_reads_total"} + +# RPC error rate for server_info +rate(rpc_method_errored_total{method="server_info"}[5m]) + +# Job queue wait time p95 +histogram_quantile(0.95, sum by (le) (rate(job_queued_us_bucket[5m]))) + +# Job run time p95 split by producer, for one job type +histogram_quantile(0.95, sum by (le, handler) (rate(job_running_us_bucket{job_type="ledgerRequest"}[5m]))) + +# TxQ utilization percentage +txq_metrics{metric="txq_count"} / txq_metrics{metric="txq_max_size"} + +# High load factor alert candidate +load_factor_metrics{metric="load_factor"} > 5 + +# Job types currently hitting their concurrency limit (backpressure). +# Scoped to one node: unscoped, this aggregates every node on the stack. +max by (__name__) ({__name__=~"jobq_.*_deferred", service_instance_id="$node"}) > 0 + +# GetObject NodeStore hit ratio +sum(rate(getobject_lookups_total{result="hit"}[5m])) + / sum(rate(getobject_lookups_total[5m])) + +# GetObject fetch-loop p95 (microseconds) +histogram_quantile(0.95, sum by (le) (rate(getobject_lookup_us_bucket[5m]))) + +# GetObject requests refused, by reason +sum by (reason) (rate(getobject_rejected_total[5m])) +``` + +### Phase 7+: External Dashboard Parity Metrics + +> **Source**: [External Dashboard Parity Spec](./06-implementation-phases.md#appendix-external-dashboard-parity) — metrics inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). +> +> **Task breakdown**: Phase 7 Tasks 7.9-7.16 (implementation), Phase 9 Tasks 9.11-9.13 (dashboards) + +These metrics fill gaps identified by comparing xrpld's internal observability with the community external dashboard's 86-metric coverage. All are exported via the OTel Metrics SDK (same `PeriodicMetricReader` as Phase 9 metrics). + +#### Validation Agreement (Observable Gauge — `validation_agreement`) + +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------------------- | ------ | -------- | --------------------------------------- | +| `validation_agreement{metric="agreement_pct_1h"}` | Double | `metric` | Rolling 1h agreement percentage (0-100) | +| `validation_agreement{metric="agreement_pct_24h"}` | Double | `metric` | Rolling 24h agreement percentage | +| `validation_agreement{metric="agreements_1h"}` | Int64 | `metric` | Agreed validations in 1h window | +| `validation_agreement{metric="missed_1h"}` | Int64 | `metric` | Missed validations in 1h window | +| `validation_agreement{metric="agreements_24h"}` | Int64 | `metric` | Agreed validations in 24h window | +| `validation_agreement{metric="missed_24h"}` | Int64 | `metric` | Missed validations in 24h window | + +Data source: `ValidationTracker` class with 8s grace period and 5m late repair window. + +#### Validator Health (Observable Gauge — `validator_health`) + +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------- | ------ | -------- | ------------------------------ | +| `validator_health{metric="amendment_blocked"}` | Int64 | `metric` | 1 if amendment-blocked, else 0 | +| `validator_health{metric="unl_blocked"}` | Int64 | `metric` | 1 if UNL-blocked, else 0 | +| `validator_health{metric="unl_expiry_days"}` | Double | `metric` | Days until UNL list expires | +| `validator_health{metric="validation_quorum"}` | Int64 | `metric` | Validation quorum threshold | + +#### Peer Quality (Observable Gauge — `peer_quality`) + +| Prometheus Metric | Type | Labels | Description | +| ------------------------------------------------- | ------ | -------- | ------------------------------------ | +| `peer_quality{metric="peer_latency_p90_ms"}` | Double | `metric` | P90 peer latency in milliseconds | +| `peer_quality{metric="peers_insane_count"}` | Int64 | `metric` | Peers with diverged tracking status | +| `peer_quality{metric="peers_higher_version_pct"}` | Double | `metric` | % of peers on newer xrpld version | +| `peer_quality{metric="upgrade_recommended"}` | Int64 | `metric` | 1 if >60% of peers are newer version | + +#### Ledger Economy (Observable Gauge — `ledger_economy`) + +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------- | ------ | -------- | ---------------------------------- | +| `ledger_economy{metric="base_fee_xrp"}` | Double | `metric` | Base transaction fee in drops | +| `ledger_economy{metric="reserve_base_xrp"}` | Double | `metric` | Account reserve in drops | +| `ledger_economy{metric="reserve_inc_xrp"}` | Double | `metric` | Owner reserve increment in drops | +| `ledger_economy{metric="ledger_age_seconds"}` | Double | `metric` | Seconds since last validated close | +| `ledger_economy{metric="transaction_rate"}` | Double | `metric` | Smoothed transaction rate (tx/s) | + +#### State Tracking (Observable Gauge — `state_tracking`) + +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------------------------- | ------ | -------- | -------------------------------------- | +| `state_tracking{metric="state_value"}` | Int64 | `metric` | Numeric state 0-6 (see encoding below) | +| `state_tracking{metric="time_in_current_state_seconds"}` | Double | `metric` | Duration in current state | + +State value encoding: 0=disconnected, 1=connected, 2=syncing, 3=tracking, 4=full, 5=validating (FULL + validating), 6=proposing (FULL + proposing). + +#### Storage Detail (Observable Gauge — `storage_detail`) + +| Prometheus Metric | Type | Labels | Description | +| ------------------------------------- | ----- | -------- | ---------------------- | +| `storage_detail{metric="nudb_bytes"}` | Int64 | `metric` | NuDB backend file size | + +#### Synchronous Counters (Phase 7+) + +| Prometheus Metric | Type | Description | Increment Site | +| --------------------------- | ------- | ---------------------------- | ---------------- | +| `ledgers_closed_total` | Counter | Ledgers closed by consensus | RCLConsensus.cpp | +| `validations_sent_total` | Counter | Validations sent | RCLConsensus.cpp | +| `validations_checked_total` | Counter | Network validations observed | LedgerMaster.cpp | +| `state_changes_total` | Counter | Operating mode transitions | NetworkOPs.cpp | + +Lifetime tallies exported as monotonic **ObservableCounters** (not synchronous +counters), observed from an existing cumulative source each collection cycle: + +| Prometheus Metric | Type | Description | Source | +| ----------------------------- | ----------------- | ------------------------------------------ | ---------------------------------------------------- | +| `validation_agreements_total` | ObservableCounter | Lifetime validations that initially agreed | ValidationTracker.cpp | +| `validation_missed_total` | ObservableCounter | Lifetime validations that initially missed | ValidationTracker.cpp | +| `jq_trans_overflow_total` | ObservableCounter | Job queue transaction overflows | Overlay::getJqTransOverflow (PeerImp.cpp increments) | + +> **Counting semantics (initial-classification only):** each reconciled ledger increments exactly +> one of these two counters, at first classification. A later late-repair (miss → agreement) does +> **not** move either counter — keeping both strictly monotonic (a Prometheus `_total` must never +> decrease) and additive (`agreements_total + missed_total` = ledgers reconciled). The +> repair-aware, windowed view remains on `validation_agreement{metric="…"}`. + +#### Span Attribute Enrichments (Phases 2-4) + +| Span Name | New Attribute | Type | Source | +| --------------------------- | ------------------------------------ | ------ | ------------------------ | +| `rpc.command.*` | `xrpl.node.amendment_blocked` | bool | Phase 2 — RPCHandler.cpp | +| `rpc.command.*` | `xrpl.node.server_state` | string | Phase 2 — RPCHandler.cpp | +| `tx.receive` | `xrpl.peer.version` | string | Phase 3 — PeerImp.cpp | +| `consensus.validation.send` | `xrpl.validation.ledger_hash` | string | Phase 4 — RCLConsensus | +| `consensus.validation.send` | `xrpl.validation.full` | bool | Phase 4 — RCLConsensus | +| `peer.validation.receive` | `xrpl.peer.validation.ledger_hash` | string | Phase 4 — PeerImp.cpp | +| `peer.validation.receive` | `xrpl.peer.validation.full` | bool | Phase 4 — PeerImp.cpp | +| `consensus.accept` | `xrpl.consensus.validation_quorum` | int64 | Phase 4 — RCLConsensus | +| `consensus.accept` | `xrpl.consensus.proposers_validated` | int64 | Phase 4 — RCLConsensus | + +### New Grafana Dashboards (Phase 9) + +| Dashboard | UID | Data Source | Key Panels | +| ------------------------------------ | ------------------ | ----------- | ------------------------------------------------------------------------------------------------------------------------ | +| Fee Market & TxQ | `fee-market` | Prometheus | TxQ depth/capacity, fee levels, load factor breakdown | +| Job Queue Analysis | `job-queue` | Prometheus | Per-job rates, queue wait times, execution times | +| RPC Performance (per-method section) | `rpc-performance` | Prometheus | Per-method call rates, error rates, latency distributions (added as a section to the existing RPC Performance dashboard) | +| Validator Health | `validator-health` | Prometheus | Agreement %, validation rate, amendment/UNL, state | +| Peer Quality | `peer-quality` | Prometheus | P90 latency, insane peers, version awareness, disconnects | + +### Updated Grafana Dashboards (Phase 9) + +| Dashboard | UID | New Panels Added | +| -------------------- | -------------------------- | -------------------------------------------------------------------- | +| Node Health (StatsD) | `xrpld-statsd-node-health` | NodeStore I/O, cache hit rates, object instance counts | +| System Node Health | `node-health` | Ledger economy row: base fee, reserves, ledger age, transaction rate | + +### New Grafana Dashboards (Phase 11) + +| Dashboard | UID | Data Source | Key Panels | +| ------------------ | --------------------------- | ----------- | ---------------------------------------------------------------------- | +| Validator Health | `validator-health` | Prometheus | Server state timeline, proposer count, converge time, amendment voting | +| Network Topology | `xrpld-network-topology` | Prometheus | Peer count, version distribution, latency distribution, diverged peers | +| Fee Market (Ext) | `xrpld-fee-market-external` | Prometheus | Fee levels, queue depth, load factor breakdown, escalation timeline | +| DEX & AMM Overview | `xrpld-dex-amm` | Prometheus | AMM TVL, order book depth, spread trends, trading fee revenue | + +### Prometheus Alerting Rules (Phase 11) + +| Alert Name | Severity | Condition | For | +| ---------------------------------- | -------- | ----------------------------------------------------------- | --- | +| `XRPLServerNotFull` | Critical | `xrpl_server_state < 4` for 15m | 15m | +| `XRPLAmendmentBlocked` | Critical | `xrpl_amendment_blocked == 1` | 1m | +| `XRPLNoPeers` | Critical | `xrpl_peers_count == 0` | 5m | +| `XRPLLedgerStale` | Critical | `xrpl_validated_ledger_age_seconds > 120` | 2m | +| `XRPLHighIOLatency` | Critical | `xrpl_io_latency_ms > 100` | 5m | +| `XRPLUnsupportedAmendmentMajority` | Critical | `xrpl_amendment_unsupported_majority == 1` | 1m | +| `XRPLLowPeerCount` | Warning | `xrpl_peers_count < 10` | 15m | +| `XRPLHighLoadFactor` | Warning | `xrpl_load_factor > 10` | 10m | +| `XRPLSlowConsensus` | Warning | `xrpl_last_close_converge_time_seconds > 6` | 5m | +| `XRPLValidatorListExpiring` | Warning | `(xrpl_validator_list_expiration_seconds - time()) < 86400` | 1h | +| `XRPLStateFlapping` | Warning | `rate(xrpl_state_transitions_total{state="full"}[1h]) > 2` | 30m | + +--- + +## 6. Known Issues + +| Issue | Impact | Status | +| ------------------------------------------------------------------ | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------- | +| `warn` and `drop` metrics use non-standard StatsD `\|m` meter type | Metrics silently dropped by OTel StatsD receiver | Phase 6 Task 6.1 — needs `\|m` → `\|c` change in StatsDCollector.cpp | +| `jobq_job_count` may not emit in standalone mode | Missing from Prometheus in some test configs | Requires active job queue activity | +| `rpc_requests` depends on `[insight]` config | Zero series if StatsD not configured | Requires `[insight] server=statsd` in xrpld.cfg | +| Peer tracing enabled by default | `peer.*` spans emit unless `trace_peer=0` | High volume — set `trace_peer=0` to opt out on busy mainnet nodes | +| `handler="other"` mixes several producers | Cannot separate `GetConsL1` from `GetConsL2` | By design — the cardinality bound; see [§Per-Job-Type Metrics](#per-job-type-metrics-synchronous-countershistogram) | + +--- + +## 7. Privacy and Data Collection + +The telemetry system is designed with privacy in mind: + +- **No private keys** are ever included in spans or metrics +- **No account balances** or financial data is traced +- **Transaction hashes** are included (public on-ledger data) but not transaction contents +- **Peer IDs** are internal identifiers, not IP addresses +- **All telemetry is opt-in** — disabled by default at build time (`-Dtelemetry=OFF`) +- **Sampling** — head sampling is fixed at 1.0 (sample everything); reduce data volume with collector-side tail sampling +- **Data stays local** — the default stack sends data to `localhost` only + +--- + +## 8. Configuration Quick Reference + +> **Full reference**: [05-configuration-reference.md](./05-configuration-reference.md) §5.1 for all `[telemetry]` options with defaults, the config parser implementation, and collector YAML configurations (dev and production). + +### Minimal Setup (development) + +```ini +[telemetry] +enabled=1 + +[insight] +server=statsd +address=127.0.0.1:8125 +prefix=xrpld +``` + +### Production Setup + +```ini +[telemetry] +enabled=1 +endpoint=http://otel-collector:4318/v1/traces +trace_peer=0 +batch_size=1024 +max_queue_size=4096 + +[insight] +server=statsd +address=otel-collector:8125 +prefix=xrpld +``` + +### Trace Category Toggle + +| Config Key | Default | Controls | +| -------------------- | ------- | ---------------------------- | +| `trace_rpc` | `1` | `rpc.*` spans | +| `trace_transactions` | `1` | `tx.*` spans | +| `trace_consensus` | `1` | `consensus.*` spans | +| `trace_ledger` | `1` | `ledger.*` spans | +| `trace_peer` | `1` | `peer.*` spans (high volume) | diff --git a/OpenTelemetryPlan/OpenTelemetryPlan.md b/OpenTelemetryPlan/OpenTelemetryPlan.md new file mode 100644 index 00000000000..b7289d5004d --- /dev/null +++ b/OpenTelemetryPlan/OpenTelemetryPlan.md @@ -0,0 +1,226 @@ +# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld + +## Executive Summary + +> **OTLP** = OpenTelemetry Protocol + +This document provides a comprehensive implementation plan for integrating OpenTelemetry distributed tracing into the xrpld XRP Ledger node software. The plan addresses the unique challenges of a decentralized peer-to-peer system where trace context must propagate across network boundaries between independent nodes. + +### Key Benefits + +- **End-to-end transaction visibility**: Track transactions from submission through consensus to ledger inclusion +- **Consensus round analysis**: Understand timing and behavior of consensus phases across validators +- **RPC performance insights**: Identify slow handlers and optimize response times +- **Network topology understanding**: Visualize message propagation patterns between peers +- **Incident debugging**: Correlate events across distributed nodes during issues + +### Estimated Performance Overhead + +| Metric | Overhead | Notes | +| ------------- | ---------- | ------------------------------------------------ | +| CPU | 1-3% | Span creation and attribute setting | +| Memory | <10 MB | SDK statics + batch buffer + worker thread stack | +| Network | 10-50 KB/s | Compressed OTLP export to collector | +| Latency (p99) | <2% | With proper sampling configuration | + +--- + +## Document Structure + +This implementation plan is organized into modular documents for easier navigation: + +
+ +```mermaid +flowchart TB + overview["📋 OpenTelemetryPlan.md
(This Document)"] + + subgraph fundamentals["Fundamentals"] + fund["00-tracing-fundamentals.md"] + end + + subgraph analysis["Analysis & Design"] + arch["01-architecture-analysis.md"] + design["02-design-decisions.md"] + end + + subgraph impl["Implementation"] + strategy["03-implementation-strategy.md"] + config["05-configuration-reference.md"] + end + + subgraph deploy["Deployment & Planning"] + phases["06-implementation-phases.md"] + backends["07-observability-backends.md"] + appendix["08-appendix.md"] + secure["secure-OTel.md"] + dataref["09-data-collection-reference.md"] + end + + overview --> fundamentals + overview --> analysis + overview --> impl + overview --> deploy + + fund --> arch + arch --> design + design --> strategy + strategy --> config + config --> phases + phases --> backends + backends --> appendix + backends --> secure + appendix --> dataref + + style overview fill:#1b5e20,stroke:#0d3d14,color:#fff,stroke-width:2px + style fundamentals fill:#00695c,stroke:#004d40,color:#fff + style fund fill:#00695c,stroke:#004d40,color:#fff + style analysis fill:#0d47a1,stroke:#082f6a,color:#fff + style impl fill:#bf360c,stroke:#8c2809,color:#fff + style deploy fill:#4a148c,stroke:#2e0d57,color:#fff + style arch fill:#0d47a1,stroke:#082f6a,color:#fff + style design fill:#0d47a1,stroke:#082f6a,color:#fff + style strategy fill:#bf360c,stroke:#8c2809,color:#fff + style config fill:#bf360c,stroke:#8c2809,color:#fff + style phases fill:#4a148c,stroke:#2e0d57,color:#fff + style backends fill:#4a148c,stroke:#2e0d57,color:#fff + style appendix fill:#4a148c,stroke:#2e0d57,color:#fff + style secure fill:#4a148c,stroke:#2e0d57,color:#fff + style dataref fill:#4a148c,stroke:#2e0d57,color:#fff +``` + +
+ +--- + +## Table of Contents + +| Section | Document | Description | +| ------- | -------------------------------------------------------------- | ---------------------------------------------------------------------- | +| **0** | [Tracing Fundamentals](./00-tracing-fundamentals.md) | Distributed tracing concepts, span relationships, context propagation | +| **1** | [Architecture Analysis](./01-architecture-analysis.md) | xrpld component analysis, trace points, instrumentation priorities | +| **2** | [Design Decisions](./02-design-decisions.md) | SDK selection, exporters, span naming, attributes, context propagation | +| **3** | [Implementation Strategy](./03-implementation-strategy.md) | Directory structure, key principles, performance optimization | +| **5** | [Configuration Reference](./05-configuration-reference.md) | xrpld config, CMake integration, Collector configurations | +| **6** | [Implementation Phases](./06-implementation-phases.md) | 5-phase timeline, tasks, risks, success metrics | +| **7** | [Observability Backends](./07-observability-backends.md) | Backend selection guide and production architecture | +| **8** | [Appendix](./08-appendix.md) | Glossary, references, version history | +| **9** | [Data Collection Reference](./09-data-collection-reference.md) | Complete inventory of spans, attributes, metrics, and dashboards | +| **Sec** | [Securing the OTel Pipeline](./secure-OTel.md) | Threat model and hardening (mTLS, peer trace-context validation) | + +--- + +## 0. Tracing Fundamentals + +This document introduces distributed tracing concepts for readers unfamiliar with the domain. It covers what traces and spans are, how parent-child and follows-from relationships model causality, how context propagates across service boundaries, and how sampling controls data volume. It also maps these concepts to xrpld-specific scenarios like transaction relay and consensus. + +➡️ **[Read Tracing Fundamentals](./00-tracing-fundamentals.md)** + +--- + +## 1. Architecture Analysis + +> **WS** = WebSocket | **TxQ** = Transaction Queue + +The xrpld node consists of several key components that require instrumentation for comprehensive distributed tracing. The main areas include the RPC server (HTTP/WebSocket), Overlay P2P network, Consensus mechanism (RCLConsensus), JobQueue for async task execution, PathFinding, Transaction Queue (TxQ), fee escalation (LoadManager), ledger acquisition, validator management, and existing observability infrastructure (PerfLog, Insight/StatsD, Journal logging). + +Key trace points span across transaction submission via RPC, peer-to-peer message propagation, consensus round execution, ledger building, path computation, transaction queue behavior, fee escalation, and validator health. The implementation prioritizes high-value, low-risk components first: RPC handlers provide immediate value with minimal risk, while consensus tracing requires careful implementation to avoid timing impacts. + +➡️ **[Read full Architecture Analysis](./01-architecture-analysis.md)** + +--- + +## 2. Design Decisions + +> **OTLP** = OpenTelemetry Protocol | **CNCF** = Cloud Native Computing Foundation + +The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/HTTP to an OpenTelemetry Collector, which provides flexible routing and sampling. OTLP/gRPC is planned future work (see design decisions §2.2.2). + +Span naming follows a hierarchical `.` convention (e.g., `rpc.submit`, `tx.relay`, `consensus.round`). Context propagation uses W3C Trace Context headers for HTTP and embedded Protocol Buffer fields for P2P messages. The implementation coexists with existing PerfLog and Insight observability systems through correlation IDs. + +**Data Collection & Privacy**: Telemetry collects only operational metadata (timing, counts, hashes) — never sensitive content (private keys, balances, amounts, raw payloads). Privacy protection includes account hashing, configurable redaction, sampling, and collector-level filtering. Node operators retain full control over telemetry configuration. + +➡️ **[Read full Design Decisions](./02-design-decisions.md)** + +--- + +## 3. Implementation Strategy + +The telemetry code is organized under `include/xrpl/telemetry/` for headers and `src/libxrpl/telemetry/` for implementation. Key principles include RAII-based span management via `SpanGuard` (with `discard()` for dropping unwanted spans), a `FilteringSpanProcessor` that intercepts `OnEnd()` to prevent discarded spans from entering the export pipeline, conditional compilation with `XRPL_ENABLE_TELEMETRY`, and minimal runtime overhead through batch processing and efficient sampling. + +Performance optimization strategies include head sampling fixed at 100% (intentionally not configurable, so trace keep/drop decisions stay coherent across nodes), tail-based sampling at the collector for errors and slow traces to reduce volume, batch export to reduce network overhead, and conditional instrumentation that compiles to no-ops when disabled. + +➡️ **[Read full Implementation Strategy](./03-implementation-strategy.md)** + +--- + +## 5. Configuration Reference + +> **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring + +Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, and component-level filtering. Head sampling is fixed at 1.0 (not operator-configurable); volume reduction is done by tail sampling in the collector. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control. + +OpenTelemetry Collector configurations are provided for development and production (with tail-based sampling, Tempo, and Elastic APM). Docker Compose examples enable quick local development environment setup. + +➡️ **[View full Configuration Reference](./05-configuration-reference.md)** + +--- + +## 6. Implementation Phases + +The implementation spans 13 weeks across 8 phases: + +| Phase | Duration | Focus | Key Deliverables | +| ----- | ----------- | --------------------- | ----------------------------------------------------------- | +| 1 | Weeks 1-2 | Core Infrastructure | SDK integration, Telemetry interface, Configuration | +| 2 | Weeks 3-4 | RPC Tracing | HTTP context extraction, Handler instrumentation | +| 3 | Weeks 5-6 | Transaction Tracing | Protocol Buffer context, Relay propagation | +| 4 | Weeks 7-8 | Consensus Tracing | Round spans, Proposal/validation tracing | +| 5 | Week 9 | Documentation | Runbook, Dashboards, Training | +| 6 | Week 10 | StatsD Metrics Bridge | OTel Collector StatsD receiver, 3 Grafana dashboards | +| 7 | Weeks 11-12 | Native OTel Metrics | OTelCollector impl, OTLP metrics export, StatsD deprecation | +| 8 | Week 13 | Log-Trace Correlation | trace_id in logs, Loki ingestion, Tempo↔Loki linking | + +**Total Effort**: 65.1 developer-days with 2 developers + +➡️ **[View full Implementation Phases](./06-implementation-phases.md)** + +--- + +## 7. Observability Backends + +> **APM** = Application Performance Monitoring | **GCS** = Google Cloud Storage + +Grafana Tempo is recommended for all environments due to its cost-effectiveness and Grafana integration, while Elastic APM is ideal for organizations with existing Elastic infrastructure. + +The recommended production architecture uses a gateway collector pattern with regional collectors performing tail-based sampling, routing traces to multiple backends (Tempo for primary storage, Elastic for log correlation, S3/GCS for long-term archive). + +➡️ **[View Observability Backend Recommendations](./07-observability-backends.md)** + +--- + +## 8. Appendix + +The appendix contains a glossary of OpenTelemetry and xrpld-specific terms, references to external documentation and specifications, version history for this implementation plan, and a complete document index. + +➡️ **[View Appendix](./08-appendix.md)** + +--- + +## 9. Data Collection Reference + +A single-source-of-truth reference documenting every piece of telemetry data collected by xrpld. Covers all 16 OpenTelemetry spans with their 22 attributes, all StatsD metrics (gauges, counters, histograms, overlay traffic), SpanMetrics-derived Prometheus metrics, and all 10 Grafana dashboards. Includes Tempo search guides and Prometheus query examples. + +➡️ **[View Data Collection Reference](./09-data-collection-reference.md)** + +--- + +## Securing the OTel Pipeline + +Threat model and hardening guidance for production deployments where xrpld nodes ship telemetry to a centrally-hosted collector across an untrusted network. Covers the two attack surfaces (collector ingress and peer trace-context spoofing) and the chosen defenses: mTLS as primary collector auth, NetworkPolicy as defense-in-depth, and source-side validation plus per-peer rate limiting for the `protocol::TraceContext` field on peer messages. + +➡️ **[View Securing the OTel Pipeline](./secure-OTel.md)** + +--- + +_This document provides a comprehensive implementation plan for integrating OpenTelemetry distributed tracing into the xrpld XRP Ledger node software. For detailed information on any section, follow the links to the corresponding sub-documents._ diff --git a/OpenTelemetryPlan/Phase10_taskList.md b/OpenTelemetryPlan/Phase10_taskList.md new file mode 100644 index 00000000000..7022652b9ed --- /dev/null +++ b/OpenTelemetryPlan/Phase10_taskList.md @@ -0,0 +1,258 @@ +# Phase 10: Synthetic Workload Generation & Telemetry Validation — Task List + +> **Status**: Future Enhancement +> +> **Goal**: Build tools that generate realistic XRPL traffic to validate the full Phases 1-9 telemetry stack end-to-end — all spans, attributes, metrics, dashboards, and log-trace correlation — under controlled load. +> +> **Scope**: Python/shell test harness + multi-node docker-compose environment + automated validation scripts + performance benchmarks. +> +> **Branch**: `pratik/otel-phase10-workload-validation` (from `pratik/otel-phase9-metric-gap-fill`) +> +> **Depends on**: Phase 9 (internal metric gap fill) — validates the full metric surface + +### Related Plan Documents + +| Document | Relevance | +| -------------------------------------------------------------------- | --------------------------------------------------------------- | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 10 plan: motivation, architecture, exit criteria (§6.8.3) | +| [09-data-collection-reference.md](./09-data-collection-reference.md) | Defines the full inventory of spans/metrics to validate | +| [Phase9_taskList.md](./Phase9_taskList.md) | Prerequisite — all internal metrics must be emitting | + +### Why This Phase Exists + +Before Phases 1-9 can be considered production-ready, we need proof that: + +1. All required spans fire with correct attributes under real transaction workloads +2. All 255+ StatsD metrics + ~50 Phase 9 metrics appear in Prometheus with non-zero values +3. Log-trace correlation (Phase 8) produces clickable trace_id links in Loki +4. All 10 Grafana dashboards render meaningful data (no empty panels) +5. Performance overhead stays within bounds (< 3% CPU, < 5MB memory) +6. The telemetry stack survives sustained load without data loss or queue backpressure + +--- + +## Task 10.1: Multi-Node Test Harness + +**Objective**: Create a docker-compose environment with 3-5 validator nodes that produces real consensus rounds. + +**What to do**: + +- Create `docker/telemetry/docker-compose.workload.yaml`: + - 5 xrpld validator nodes with UNL configured for each other + - All telemetry enabled: `[telemetry] enabled=1`, `[insight] server=otel` + - Full OTel stack: Collector, Tempo, Prometheus, Loki, Grafana + - Shared network with service discovery + +- Each node should: + - Generate validator keys at startup + - Configure all 5 nodes in its UNL + - Enable all trace categories including `trace_peer=1` + - Write logs to a file tailed by the OTel Collector filelog receiver + +- Include a `Makefile` target: `make telemetry-workload-up` / `make telemetry-workload-down` + +**Key files**: + +- New: `docker/telemetry/docker-compose.workload.yaml` +- New: `docker/telemetry/workload/generate-validator-keys.sh` +- New: `docker/telemetry/workload/xrpld-validator.cfg.template` + +--- + +## Task 10.2: RPC Load Generator + +**Objective**: Configurable tool that fires all traced RPC commands at controlled rates. + +**What to do**: + +- Create `docker/telemetry/workload/rpc_load_generator.py`: + - Connects to one or more xrpld WebSocket endpoints + - Fires all RPC commands that have trace spans: `server_info`, `ledger`, `tx`, `account_info`, `account_lines`, `fee`, `submit`, etc. + - Configurable parameters: rate (RPS), duration, command distribution weights + - Injects `traceparent` HTTP headers to test W3C context propagation + - Logs progress and errors to stdout + +- Command distribution should match realistic production ratios: + - 40% `server_info` / `fee` (health checks) + - 30% `account_info` / `account_lines` / `account_objects` (wallet queries) + - 15% `ledger` / `ledger_data` (explorer queries) + - 10% `tx` / `account_tx` (transaction lookups) + - 5% `book_offers` / `amm_info` (DEX queries) + +**Key files**: + +- New: `docker/telemetry/workload/rpc_load_generator.py` +- New: `docker/telemetry/workload/requirements.txt` + +--- + +## Task 10.3: Transaction Submitter + +**Objective**: Generate diverse transaction types to exercise `tx.*` and `ledger.*` spans. + +**What to do**: + +- Create `docker/telemetry/workload/tx_submitter.py`: + - Pre-funds test accounts from genesis account + - Submits a mix of transaction types: + - `Payment` (XRP and issued currencies) — exercises `tx.process`, `tx.apply` + - `OfferCreate` / `OfferCancel` — DEX activity + - `TrustSet` — trust line creation for issued currencies + - `NFTokenMint` / `NFTokenCreateOffer` / `NFTokenAcceptOffer` — NFT activity + - `EscrowCreate` / `EscrowFinish` — escrow lifecycle + - `AMMCreate` / `AMMDeposit` / `AMMWithdraw` — AMM pool operations (if amendment enabled) + - Configurable: TPS target, transaction mix weights, duration + - Monitors submission results and tracks success/failure rates + +- The transaction mix ensures the telemetry captures the full range of ledger activity that third parties care about. + +**Key files**: + +- New: `docker/telemetry/workload/tx_submitter.py` +- New: `docker/telemetry/workload/test_accounts.json` (pre-generated keypairs) + +--- + +## Task 10.4: Telemetry Validation Suite + +**Objective**: Automated scripts that verify all expected telemetry data exists after a workload run. + +**What to do**: + +- Create `docker/telemetry/workload/validate_telemetry.py`: + + **Span validation** (queries Tempo API): + - Assert all required span names appear in traces (conditional spans — `grpc.*`, + `ledger.acquire`, `txq.*`, `consensus.mode_change` — are marked `optional` and + skipped when not exercised by the workload) + - Assert each span has its required attributes (bare/underscore keys per the + 2026-05-13 span-attr naming redesign; dotted `xrpl.*` reserved for resource attrs) + - Assert parent-child relationships are correct (`rpc.ws_message` → `rpc.process` → `rpc.command.*`) + - Assert span durations are reasonable (> 0, < 60s) + + **Metric validation** (queries Prometheus API): + - Assert all SpanMetrics-derived metrics are non-zero: `traces_span_metrics_calls_total`, `traces_span_metrics_duration_milliseconds_bucket` + - Assert all StatsD metrics are non-zero: `xrpld_LedgerMaster_Validated_Ledger_Age`, `xrpld_Peer_Finder_Active_*`, etc. + - Assert all Phase 9 metrics are non-zero: `xrpld_nodestore_*`, `xrpld_cache_*`, `xrpld_txq_*`, `xrpld_rpc_method_*`, `xrpld_object_count`, `xrpld_load_factor*` + - Assert metric label cardinality is within bounds + + **Log-trace correlation validation** (queries Loki API): + - Assert logs contain `trace_id=` and `span_id=` fields + - Pick a random trace_id from Tempo → query Loki for matching logs → assert results exist + - Assert Grafana derived field links are functional + + **Dashboard validation**: + - For each of the 10 Grafana dashboards, query the dashboard API and assert no panels show "No data" + +- Output: JSON report with pass/fail per check, suitable for CI. + +**Key files**: + +- New: `docker/telemetry/workload/validate_telemetry.py` +- New: `docker/telemetry/workload/expected_spans.json` (span inventory for validation) +- New: `docker/telemetry/workload/expected_metrics.json` (metric inventory for validation) + +--- + +## Task 10.5: Performance Benchmark Suite + +**Objective**: Measure CPU/memory/latency overhead of the telemetry stack. + +**What to do**: + +- Create `docker/telemetry/workload/benchmark.sh`: + - **Baseline run**: Start cluster with `[telemetry] enabled=0`, run transaction workload for 5 minutes, record metrics + - **Telemetry run**: Start cluster with full telemetry enabled, run identical workload, record metrics + - **Comparison**: Calculate deltas for: + - CPU usage (per-node average) + - Memory RSS (per-node peak) + - RPC p99 latency + - Transaction throughput (TPS) + - Consensus round time p95 + - Ledger close time p95 + +- Output: Markdown table comparing baseline vs. telemetry, with pass/fail against targets: + - CPU overhead < 3% + - Memory overhead < 5MB + - RPC latency impact < 2ms p99 + - Throughput impact < 5% + - Consensus impact < 1% + +- Store results in `docker/telemetry/workload/benchmark-results/` for historical tracking. + +**Key files**: + +- New: `docker/telemetry/workload/benchmark.sh` +- New: `docker/telemetry/workload/collect_system_metrics.sh` + +--- + +## Task 10.6: CI Integration + +**Objective**: Wire the validation suite into CI for regression detection. + +**What to do**: + +- Create a CI workflow (GitHub Actions or equivalent) that: + 1. Builds xrpld with `-DXRPL_ENABLE_TELEMETRY=ON` + 2. Starts the multi-node workload harness + 3. Runs the RPC load generator + transaction submitter for 2 minutes + 4. Runs the validation suite + 5. Runs the benchmark suite + 6. Fails the build if any validation check fails or benchmark exceeds thresholds + 7. Archives the validation report and benchmark results as artifacts + +- This should be a separate workflow (not part of the main CI), triggered manually or on telemetry-related branch changes. + +**Key files**: + +- New: `.github/workflows/telemetry-validation.yml` +- New: `docker/telemetry/workload/run-full-validation.sh` (orchestrator script) + +--- + +## Task 10.7: Documentation + +**Objective**: Document the workload tools and validation process. + +**What to do**: + +- Create `docker/telemetry/workload/README.md`: + - Quick start guide for running workload harness + - Configuration options for load generator and tx submitter + - How to read validation reports + - How to run benchmarks and interpret results + +- Update `docs/telemetry-runbook.md`: + - Add "Validating Telemetry Stack" section + - Add "Performance Benchmarking" section + +- Update `OpenTelemetryPlan/09-data-collection-reference.md`: + - Add "Validation" section with expected metric/span counts + +--- + +## Exit Criteria — Delivered in PR #6519 + +- [x] Multi-node validator cluster starts and reaches consensus +- [x] RPC load generator fires all traced RPC commands at configurable rates +- [x] Transaction submitter generates 6+ transaction types at configurable TPS +- [x] Validation suite confirms all required spans, attributes, and metrics +- [x] Log-trace correlation validated end-to-end (Loki ↔ Tempo) +- [x] Grafana dashboards render data (no empty panels) +- [x] Overhead benchmark (`benchmark.sh`) measures telemetry-off vs telemetry-on deltas +- [x] CI workflow runs validation on telemetry branch changes +- [x] Validation report output is CI-parseable (JSON with exit codes) +- [x] OTel-driven regression gate captures per-span/per-RPC/per-job timings from + Prometheus and compares against a committed baseline + +## Follow-up Work (tracked in separate PRs) + +- [ ] FU-2: Automate baseline persistence across CI runs (artifact uploaded + on merge to `develop`, downloaded on PR runs). Current mechanism + requires a manual baseline-refresh PR. +- [ ] FU-4: Replace the proxy measurements in `benchmark.sh` (wall-clock curl + p99, ledger-cadence-as-TPS, ledger-cadence-as-consensus-p95) with + PromQL quantile queries from the same pipeline the regression gate uses. +- [ ] FU-6: Grafana dashboard plotting historical baseline values keyed by + commit SHA, for triaging noisy regressions. diff --git a/OpenTelemetryPlan/Phase11_taskList.md b/OpenTelemetryPlan/Phase11_taskList.md new file mode 100644 index 00000000000..7429063984a --- /dev/null +++ b/OpenTelemetryPlan/Phase11_taskList.md @@ -0,0 +1,544 @@ +# Phase 11: Third-Party Data Collection Pipelines — Task List + +> **Status**: Future Enhancement +> +> **Goal**: Build a custom OTel Collector receiver that periodically polls xrpld's admin RPCs and exports structured metrics for external consumers — making all XRPL health, validator, peer, fee, and DEX data available as Prometheus/OTLP metrics without xrpld code changes. +> +> **Scope**: Go-based OTel Collector receiver plugin + Grafana dashboards + Prometheus alerting rules. +> +> **Branch**: `pratik/otel-phase11-third-party-collection` (from `pratik/otel-phase10-workload-validation`) +> +> **Depends on**: Phase 10 (validation harness for testing the new receiver) + +### Related Plan Documents + +| Document | Relevance | +| -------------------------------------------------------------------- | --------------------------------------------------------------- | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 11 plan: motivation, architecture, exit criteria (§6.8.4) | +| [09-data-collection-reference.md](./09-data-collection-reference.md) | Defines full metric inventory including third-party metrics | +| [Phase10_taskList.md](./Phase10_taskList.md) | Prerequisite — validation harness for testing | + +### Third-Party Consumer Gap Analysis + +This phase addresses the cross-cutting gap identified during research: **xrpld has no native Prometheus/OTLP metrics export for data accessible only via RPC**. Every consumer (exchanges, payment processors, analytics providers, validators, researchers, compliance firms, custodians) must build custom JSON-RPC polling and conversion. This receiver centralizes that work. + +| Consumer Category | Data Unlocked by This Phase | +| -------------------------- | ------------------------------------------------------------------ | +| **Exchanges** | Real-time fee estimates, TxQ capacity, server health scores | +| **Payment Processors** | Settlement latency percentiles, corridor health, path availability | +| **Analytics Providers** | Validator metrics, network topology, amendment voting status | +| **DeFi / AMM** | AMM pool TVL, DEX order book depth, trade volumes | +| **Validators / Operators** | Per-peer latency, version distribution, UNL health, alerting | +| **Compliance** | Transaction volume trends, network growth metrics | +| **Academic Researchers** | Consensus performance time-series, decentralization metrics | +| **CBDC / Tokenization** | Token supply tracking, trust line adoption, freeze status | +| **Institutional Custody** | Multi-sig status, escrow tracking, reserve calculations | +| **Wallet Providers** | Server health for node selection, fee prediction data | + +--- + +## Task 11.1: OTel Collector Receiver Scaffold + +**Objective**: Create the Go project structure for a custom OTel Collector receiver that polls xrpld JSON-RPC. + +**What to do**: + +- Create `docker/telemetry/otel-rippled-receiver/`: + - `receiver.go` — implements `receiver.Metrics` interface + - `config.go` — configuration struct (endpoint, poll interval, enabled RPCs) + - `factory.go` — receiver factory registration + - `go.mod` / `go.sum` — Go module with OTel Collector SDK dependency + +- Configuration model: + + ```yaml + xrpld_receiver: + endpoint: "http://localhost:5005" # xrpld admin RPC + poll_interval: 30s # how often to poll + enabled_collectors: + - server_info + - get_counts + - fee + - peers + - validators + - feature + - server_state + amm_pools: [] # optional: AMM pool IDs to track + book_offers_pairs: [] # optional: currency pairs for DEX depth + ``` + +- Build a custom OTel Collector binary that includes this receiver alongside the standard receivers. + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/receiver.go` +- New: `docker/telemetry/otel-rippled-receiver/config.go` +- New: `docker/telemetry/otel-rippled-receiver/factory.go` +- New: `docker/telemetry/otel-rippled-receiver/go.mod` +- New: `docker/telemetry/otel-rippled-receiver/Dockerfile` + +--- + +## Task 11.2: server_info / server_state Collector + +**Objective**: Poll `server_info` and `server_state` and export all fields as OTel metrics. + +**What to do**: + +- Implement `serverInfoCollector` that calls `server_info` (admin) and extracts: + + **Node Health Gauges:** + - `xrpl_server_state` (enum → int: disconnected=0, connected=1, syncing=2, tracking=3, full=4, proposing=5) + - `xrpl_server_state_duration_seconds` + - `xrpl_uptime_seconds` + - `xrpl_io_latency_ms` + - `xrpl_amendment_blocked` (0 or 1) + - `xrpl_peers_count` + - `xrpl_peer_disconnects_total` + - `xrpl_peer_disconnects_resources_total` + - `xrpl_jq_trans_overflow_total` + + **Consensus Gauges:** + - `xrpl_last_close_proposers` + - `xrpl_last_close_converge_time_seconds` + - `xrpl_validation_quorum` + + **Ledger Gauges:** + - `xrpl_validated_ledger_seq` + - `xrpl_validated_ledger_age_seconds` + - `xrpl_validated_ledger_base_fee_drops` + - `xrpl_validated_ledger_reserve_base_drops` + - `xrpl_validated_ledger_reserve_inc_drops` + - `xrpl_close_time_offset_seconds` (0 when absent) + + **Load Factor Gauges:** + - `xrpl_load_factor` + - `xrpl_load_factor_server` + - `xrpl_load_factor_fee_escalation` + - `xrpl_load_factor_fee_queue` + - `xrpl_load_factor_local` + - `xrpl_load_factor_net` + - `xrpl_load_factor_cluster` + + **State Accounting Gauges** (per state: disconnected, connected, syncing, tracking, full): + - `xrpl_state_duration_seconds{state=""}` + - `xrpl_state_transitions_total{state=""}` + + **Validator Info** (when node is a validator): + - `xrpl_validator_list_count` + - `xrpl_validator_list_expiration_seconds` (epoch) + - `xrpl_validator_list_active` (0 or 1) + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/collectors/server_info.go` + +--- + +## Task 11.3: get_counts Collector + +**Objective**: Poll `get_counts` and export internal object counts and NodeStore stats. + +**What to do**: + +- Implement `getCountsCollector`: + + **Database Gauges:** + - `xrpl_db_size_kb{db="total"}`, `xrpl_db_size_kb{db="ledger"}`, `xrpl_db_size_kb{db="transaction"}` + + **NodeStore Gauges:** + - `xrpl_nodestore_reads_total`, `xrpl_nodestore_reads_hit`, `xrpl_nodestore_writes_total` + - `xrpl_nodestore_read_bytes`, `xrpl_nodestore_written_bytes` + - `xrpl_nodestore_read_duration_us`, `xrpl_nodestore_write_load` + - `xrpl_nodestore_read_queue`, `xrpl_nodestore_read_threads_running` + + **Cache Gauges:** + - `xrpl_cache_hit_rate{cache="SLE"}`, `xrpl_cache_hit_rate{cache="ledger"}`, `xrpl_cache_hit_rate{cache="accepted_ledger"}` + - `xrpl_cache_size{cache="treenode"}`, `xrpl_cache_size{cache="fullbelow"}`, `xrpl_cache_size{cache="accepted_ledger"}` + + **Object Count Gauges:** + - `xrpl_object_count{type=""}` for each counted object type (Transaction, Ledger, NodeObject, STTx, STLedgerEntry, InboundLedger, Pathfinder, etc.) + + **Rates:** + - `xrpl_historical_fetch_per_minute` + - `xrpl_local_txs` + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/collectors/get_counts.go` + +--- + +## Task 11.4: Peer Topology Collector + +**Objective**: Poll `peers` and export per-peer and aggregate network metrics. + +**What to do**: + +- Implement `peersCollector`: + + **Aggregate Gauges:** + - `xrpl_peers_inbound_count` + - `xrpl_peers_outbound_count` + - `xrpl_peers_cluster_count` + + **Per-Peer Gauges** (with labels `peer_key` truncated to 8 chars for cardinality control): + - `xrpl_peer_latency_ms{peer="", version="", inbound=""}` + - `xrpl_peer_uptime_seconds{peer=""}` + - `xrpl_peer_load{peer=""}` + + **Distribution Gauges** (aggregated across all peers): + - `xrpl_peer_latency_p50_ms`, `xrpl_peer_latency_p95_ms`, `xrpl_peer_latency_p99_ms` + - `xrpl_peer_version_count{version=""}` — count of peers per software version + + **Tracking Status:** + - `xrpl_peer_diverged_count` — peers with `track=diverged` + - `xrpl_peer_unknown_count` — peers with `track=unknown` + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/collectors/peers.go` + +**Cardinality note**: Per-peer metrics use truncated keys. For large peer sets (50+), the aggregate distribution gauges are preferred over per-peer labels. + +--- + +## Task 11.5: Validator & Amendment Collector + +**Objective**: Poll `validators` and `feature` to export validator health and amendment voting status. + +**What to do**: + +- Implement `validatorCollector`: + + **From `validators` RPC:** + - `xrpl_trusted_validators_count` + - `xrpl_validator_signing` (0 or 1 — whether local validator is signing) + + **From `feature` RPC:** + - `xrpl_amendment_enabled_count` — total enabled amendments + - `xrpl_amendment_majority_count` — amendments with majority but not yet enabled + - `xrpl_amendment_vetoed_count` — locally vetoed amendments + - `xrpl_amendment_unsupported_majority` (0 or 1) — any unsupported amendment has majority (critical alert) + + **Per-amendment with majority** (limited cardinality — only amendments with `majority` set): + - `xrpl_amendment_majority_time{name=""}` — epoch time when majority was gained + - `xrpl_amendment_votes{name=""}` — current vote count + - `xrpl_amendment_threshold{name=""}` — votes needed + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/collectors/validators.go` + +--- + +## Task 11.6: Fee & TxQ Collector + +**Objective**: Poll `fee` RPC and export real-time fee market data. + +**What to do**: + +- Implement `feeCollector` that calls the public `fee` RPC: + + **Fee Level Gauges:** + - `xrpl_fee_current_ledger_size` — transactions in current open ledger + - `xrpl_fee_expected_ledger_size` — expected transactions at close + - `xrpl_fee_max_queue_size` — maximum transaction queue size + - `xrpl_fee_open_ledger_fee_drops` — minimum fee for open ledger inclusion + - `xrpl_fee_median_fee_drops` — median fee level + - `xrpl_fee_minimum_fee_drops` — base reference fee + - `xrpl_fee_queue_size` — current queue depth + +- This overlaps with Phase 9's internal TxQ metrics but provides an external-only collection path that doesn't require xrpld code changes. + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/collectors/fee.go` + +--- + +## Task 11.7: DEX & AMM Collector (Optional) + +**Objective**: Periodically poll configured AMM pools and order book pairs for DeFi metrics. + +**What to do**: + +- Implement `dexCollector` (enabled only when `amm_pools` or `book_offers_pairs` are configured): + + **AMM Pool Gauges** (per configured pool): + - `xrpl_amm_reserve{pool="", asset=""}` — pool reserve amount + - `xrpl_amm_lp_token_supply{pool=""}` — outstanding LP tokens + - `xrpl_amm_trading_fee{pool=""}` — pool trading fee (basis points) + - `xrpl_amm_tvl_drops{pool=""}` — total value locked (XRP-denominated) + + **Order Book Gauges** (per configured pair): + - `xrpl_orderbook_bid_depth{pair="/"}` — total bid volume + - `xrpl_orderbook_ask_depth{pair="/"}` — total ask volume + - `xrpl_orderbook_spread{pair="/"}` — best bid-ask spread + - `xrpl_orderbook_offer_count{pair="/", side="bid|ask"}` — number of offers + +**Key files**: + +- New: `docker/telemetry/otel-rippled-receiver/collectors/dex.go` + +**Note**: This is optional because it requires explicit configuration of which pools/pairs to track. Default configuration tracks no DEX data. + +--- + +## Task 11.8: Prometheus Alerting Rules + +**Objective**: Create production-ready alerting rules for the metrics exported by this receiver. + +**What to do**: + +- Create `docker/telemetry/prometheus/rippled-alerts.yml`: + + **Tier 1 — Critical (page immediately):** + + ```yaml + - alert: XRPLServerNotFull + expr: xrpl_server_state < 4 + for: 15m + + - alert: XRPLAmendmentBlocked + expr: xrpl_amendment_blocked == 1 + for: 1m + + - alert: XRPLNoPeers + expr: xrpl_peers_count == 0 + for: 5m + + - alert: XRPLLedgerStale + expr: xrpl_validated_ledger_age_seconds > 120 + for: 2m + + - alert: XRPLHighIOLatency + expr: xrpl_io_latency_ms > 100 + for: 5m + + - alert: XRPLUnsupportedAmendmentMajority + expr: xrpl_amendment_unsupported_majority == 1 + for: 1m + ``` + + **Tier 2 — Warning (investigate within hours):** + + ```yaml + - alert: XRPLLowPeerCount + expr: xrpl_peers_count < 10 + for: 15m + + - alert: XRPLHighLoadFactor + expr: xrpl_load_factor > 10 + for: 10m + + - alert: XRPLSlowConsensus + expr: xrpl_last_close_converge_time_seconds > 6 + for: 5m + + - alert: XRPLValidatorListExpiring + expr: (xrpl_validator_list_expiration_seconds - time()) < 86400 + for: 1h + + - alert: XRPLClockDrift + expr: xrpl_close_time_offset_seconds > 0 + for: 5m + + - alert: XRPLStateFlapping + expr: rate(xrpl_state_transitions_total{state="full"}[1h]) > 2 + for: 30m + ``` + +**Key files**: + +- New: `docker/telemetry/prometheus/rippled-alerts.yml` +- Update: `docker/telemetry/prometheus/prometheus.yml` (add rule_files reference) + +--- + +## Task 11.9: New Grafana Dashboards + +**Objective**: Create 4 new dashboards for the data exported by the receiver. + +**What to do**: + +- **Validator Health** (`validator-health`): + - Server state timeline, state duration breakdown + - Proposer count trend, converge time trend, validation quorum + - Validator list expiration countdown + - Amendment voting status (majority/enabled/vetoed) + +- **Network Topology** (`xrpld-network-topology`): + - Peer count (inbound/outbound/cluster), peer version distribution + - Peer latency distribution (p50/p95/p99), diverged peer count + - Geographic distribution (if enriched with GeoIP) + - Peer uptime distribution + +- **Fee Market** (`xrpld-fee-market-external`): + - Current fee levels (open ledger, median, minimum), fee escalation timeline + - Queue depth vs. capacity, transactions per ledger + - Load factor breakdown (server/network/cluster/escalation) + +- **DEX & AMM Overview** (`xrpld-dex-amm`) (only populated when DEX collectors are configured): + - AMM pool TVL, reserve ratios, LP token supply + - Order book depth per pair, spread trends + - Trading fee revenue estimates + +**Key files**: + +- New: `docker/telemetry/grafana/dashboards/rippled-validator-health.json` +- New: `docker/telemetry/grafana/dashboards/rippled-network-topology.json` +- New: `docker/telemetry/grafana/dashboards/rippled-fee-market-external.json` +- New: `docker/telemetry/grafana/dashboards/rippled-dex-amm.json` + +--- + +## Task 11.10: Integration with Phase 10 Validation + +**Objective**: Extend the Phase 10 validation suite to verify this receiver's metrics. + +**What to do**: + +- Update `docker/telemetry/workload/validate_telemetry.py`: + - Add assertions for all `xrpl_*` metrics produced by the receiver + - Verify metric labels have expected values + - Verify alerting rules fire correctly (inject a "bad" state and check alert) + +- Update `docker/telemetry/docker-compose.workload.yaml`: + - Add the custom OTel Collector build with the xrpld receiver + - Configure the receiver to poll one of the test nodes + +**Key files**: + +- Update: `docker/telemetry/workload/validate_telemetry.py` +- Update: `docker/telemetry/docker-compose.workload.yaml` +- Update: `docker/telemetry/workload/expected_metrics.json` + +--- + +## Task 11.11: Documentation + +**Objective**: Document the receiver, its metrics, deployment, and alerting. + +**What to do**: + +- Create `docker/telemetry/otel-rippled-receiver/README.md`: + - Architecture overview (how the receiver fits into the OTel Collector) + - Configuration reference (all config options with defaults) + - Metric reference table (all exported metrics with types and labels) + - Deployment guide (building custom collector binary, docker-compose integration) + +- Update `OpenTelemetryPlan/09-data-collection-reference.md`: + - Add "Third-Party Metrics (OTel Collector Receiver)" section + - Add new Grafana dashboard reference (4 dashboards) + - Add alerting rules reference + +- Update `docs/telemetry-runbook.md`: + - Add "Third-Party Metrics Receiver" troubleshooting section + - Add alerting playbook (what to do for each Tier 1/Tier 2 alert) + +--- + +## Task 11.12: Alert Rules for External Dashboard Parity Metrics + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — 18 alert rules ported from the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). +> +> **Upstream**: Phase 7 Tasks 7.9-7.16 (metrics), Phase 9 Tasks 9.11-9.13 (dashboards). +> **Downstream**: None — terminal task in the parity chain. + +**Objective**: Add Grafana alerting rules for the Phase 7+ parity metrics (validation agreement, validator health, peer quality, state tracking, ledger economy). These complement Task 11.8's `xrpl_*` alerts by covering the `xrpld_*` internal metrics. + +**Critical Group** (8 rules, eval interval 10s): + +| Rule | Condition | For | +| ------------------- | ------------------------------------------------------------- | --- | +| Agreement Below 90% | `xrpld_validation_agreement{metric="agreement_pct_24h"} < 90` | 30s | +| Not Proposing | `xrpld_state_tracking{metric="state_value"} < 6` | 10s | +| Unhealthy State | `xrpld_state_tracking{metric="state_value"} < 4` | 10s | +| Amendment Blocked | `xrpld_validator_health{metric="amendment_blocked"} == 1` | 1m | +| UNL Expiring | `xrpld_validator_health{metric="unl_expiry_days"} < 14` | 1h | +| High IO Latency | `histogram_quantile(0.95, xrpld_ios_latency_bucket) > 50` | 1m | +| High Load Factor | `xrpld_load_factor_metrics{metric="load_factor"} > 1000` | 1m | +| Peer Count Critical | `xrpld_server_info{metric="peers"} < 5` | 1m | + +**Network Group** (3 rules, eval interval 10s): + +| Rule | Condition | For | +| ------------------------- | ----------------------------------------------------------------- | --- | +| Peer Drop >10% | `delta(xrpld_server_info{metric="peers"}[30s]) / ... * 100 < -10` | 30s | +| Peer Drop >30% | Same formula, threshold -30 | 30s | +| P90 Latency + Disconnects | `peer_latency_p90_ms > 500 AND rate(disconnects) > 0` | 2m | + +**Performance Group** (7 rules, eval interval 10s): + +| Rule | Condition | For | +| ------------------- | ------------------------------------------------------------ | --- | +| CPU High | Per-core CPU > 80% (requires node_exporter) | 2m | +| Memory Critical | Memory usage > 90% (requires node_exporter) | 1m | +| Disk Warning | Disk usage > 85% (requires node_exporter) | 2m | +| Job Queue Overflow | `rate(xrpld_jq_trans_overflow_total[5m]) > 0` | 1m | +| Upgrade Recommended | `xrpld_peer_quality{metric="peers_higher_version_pct"} > 60` | 1m | +| TX Rate Drop | Transaction rate dropped > 50% in 5m window | 5m | +| Stale Ledger | `xrpld_ledger_economy{metric="ledger_age_seconds"} > 30` | 1m | + +**Notification channel templates**: Email/SMTP, Discord, Slack, PagerDuty. + +**Key files**: + +- New/extend: `docker/telemetry/grafana/alerting/alert-rules-parity.yaml` +- New: `docker/telemetry/grafana/alerting/contact-points.yaml` (template configs) +- New: `docker/telemetry/grafana/alerting/notification-policies.yaml` + +**Exit Criteria**: + +- [ ] All 18 rules evaluate without errors in Grafana alerting UI +- [ ] Critical rules fire within expected timeframe when conditions are met +- [ ] Notification channel templates are documented (not hard-coded to any service) + +--- + +## Task 11.13: Dual-Datasource Architecture Documentation + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Document the external dashboard's "fast path" pattern as a future optimization for real-time panels. + +**Pattern**: A lightweight Prometheus scrape endpoint (separate from OTLP pipeline) that polls critical metrics every 2-5s, bypassing the 10s OTLP metric reader interval and Prometheus scrape interval. + +**Use case**: Real-time state panels (server state, ledger age, peer count) where 10-15s latency is too slow for operational dashboards. + +**Decision**: Document as a future option, not implement now. The current 10s interval is acceptable for v1. The external dashboard achieves 2-5s freshness by polling RPC directly, which is what the Phase 11 receiver already does. Adding a separate scrape endpoint to xrpld would only be needed if sub-second metric freshness is required from the internal metrics pipeline. + +**What to document**: + +- Architecture comparison: OTLP pipeline (10-15s) vs. direct scrape (2-5s) vs. push gateway +- When to consider: operator feedback indicating 10s is insufficient for alerting SLOs +- How to implement if needed: add `/metrics` HTTP endpoint to xrpld with Prometheus client library +- Trade-offs: additional port, additional dependency, duplication with OTLP metrics + +**Key files**: + +- Update: `OpenTelemetryPlan/09-data-collection-reference.md` (add "Future: Dual-Datasource Architecture" section) +- Update: `docs/telemetry-runbook.md` (add brief note in performance tuning section) + +**Exit Criteria**: + +- [ ] Architecture comparison documented with clear trade-offs +- [ ] Decision rationale recorded (why deferred, when to revisit) + +--- + +## Exit Criteria + +- [ ] Custom OTel Collector receiver builds and starts without errors +- [ ] All `xrpl_*` metrics from server_info, get_counts, peers, validators, fee appear in Prometheus +- [ ] Metrics update at configured poll interval (default 30s) +- [ ] 4 new Grafana dashboards operational with data +- [ ] Prometheus alerting rules fire correctly for simulated failure conditions +- [ ] DEX/AMM collector works when configured (optional — not required for base exit criteria) +- [ ] Phase 10 validation suite passes with receiver metrics included +- [ ] Receiver handles xrpld restart/unavailability gracefully (no crash, logs warning, retries) +- [ ] Documentation complete: receiver README, metric reference, alerting playbook +- [ ] Go receiver has unit tests with >80% coverage +- [ ] 18 Grafana alert rules for Phase 7+ parity metrics evaluate correctly (Task 11.12) +- [ ] Dual-datasource architecture documented with trade-offs (Task 11.13) diff --git a/OpenTelemetryPlan/Phase2_taskList.md b/OpenTelemetryPlan/Phase2_taskList.md new file mode 100644 index 00000000000..89d9ddd6e03 --- /dev/null +++ b/OpenTelemetryPlan/Phase2_taskList.md @@ -0,0 +1,240 @@ +# Phase 2: RPC Tracing Completion Task List + +> **Goal**: Complete RPC tracing coverage with unit tests, Grafana search filters, PathFind instrumentation, and config hardening. Build on the Phase 1c SpanGuard factory foundation to achieve production-quality RPC observability. +> +> **Scope**: Unit tests for core telemetry, Grafana Tempo search filters, PathFind RPC tracing, config validation (`std::clamp`). +> +> **Branch**: `pratik/otel-phase2-rpc-tracing` (from `pratik/otel-phase1c-rpc-integration`) + +### Related Plan Documents + +| Document | Relevance | +| ------------------------------------------------------------ | ------------------------------------------------------------- | +| [04-code-samples.md](./04-code-samples.md) | TraceContextPropagator (§4.4.2), RPC instrumentation (§4.5.3) | +| [02-design-decisions.md](./02-design-decisions.md) | W3C Trace Context (§2.5), span attributes (§2.4.2) | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 2 tasks (§6.3), definition of done (§6.11.2) | + +--- + +## Task 2.1: W3C Trace Context HTTP Header Extraction + +**Status**: DEFERRED → Phase 3 + +**Reason**: W3C context propagation (`traceparent`/`tracestate` headers) requires a consumer — in Phase 2, RPC spans are entirely local to the node. Phase 3 introduces cross-node transaction tracing via protobuf context propagation, which is the first use case for extracted trace context. Implementing it here without a consumer would be dead code. + +**Implemented in**: `pratik/otel-phase3-tx-tracing` — `TraceContextPropagator.h/.cpp` + +--- + +## Task 2.2: Per-Category Span Creation + +**Status**: COMPLETE (superseded by Phase 1c design) + +**Original plan**: Add `XRPL_TRACE_PEER` and `XRPL_TRACE_LEDGER` macros. + +**Actual implementation**: Phase 1c replaced all tracing macros with the `SpanGuard::span(TraceCategory, prefix, name)` factory pattern. The `TraceCategory` enum (`Rpc`, `Transactions`, `Consensus`, `Peer`, `Ledger`) serves the same conditional-creation purpose without macros. No separate task needed — the factory already supports all categories. + +--- + +## Task 2.3: Add shouldTraceLedger() to Telemetry Interface + +**Objective**: The `Setup` struct has a `traceLedger` field but there's no corresponding virtual method. Add it for interface completeness. + +**What to do**: + +- Edit `include/xrpl/telemetry/Telemetry.h`: + - Add `virtual bool shouldTraceLedger() const = 0;` + +- Update all implementations: + - `src/libxrpl/telemetry/Telemetry.cpp` (TelemetryImpl, NullTelemetryOtel) + - `src/libxrpl/telemetry/NullTelemetry.cpp` (NullTelemetry) + +**Key modified files**: + +- `include/xrpl/telemetry/Telemetry.h` +- `src/libxrpl/telemetry/Telemetry.cpp` +- `src/libxrpl/telemetry/NullTelemetry.cpp` + +--- + +## Task 2.4: Unit Tests for Core Telemetry Infrastructure + +**Status**: COMPLETE + +**Objective**: Add unit tests for the core telemetry abstractions to validate correctness and catch regressions. + +**Implemented**: + +- `src/tests/libxrpl/telemetry/TelemetryConfig.cpp`: + - Test Setup defaults (all fields have correct initial values) + - Test `setupTelemetry` config parser (empty section, full section, edge cases) + - Test `samplingRatio` clamping (values outside 0.0-1.0) + +- `src/tests/libxrpl/telemetry/SpanGuardFactory.cpp`: + - Test null guard methods are safe (setAttribute, setOk, setError, addEvent on null) + - Test category span returns null when telemetry disabled + - Test child/linked span null when no parent context + - Test move construction transfers ownership + - Test recordException safe on null guard + - Test discard() safe on null guard + +- `src/tests/libxrpl/telemetry/main.cpp` — GTest runner +- `src/tests/libxrpl/CMakeLists.txt` — test target with optional OTel linking + +--- + +## Task 2.5: Enhance RPC Span Attributes + +**Status**: DEFERRED (low priority) + +**Reason**: The high-value attributes (`command`, `version`, `role`, `status`) are already set by Phase 1c. The remaining HTTP transport-level attributes (`http.method`, `net.peer.ip`, `http.status_code`) provide limited additional insight since: + +- `http.method` is always POST for JSON-RPC +- `net.peer.ip` is debug-level info available in logs +- `duration_ms` is redundant with span duration (OTel captures start/end time natively) + +These can be added later if dashboard queries specifically need them. The node health attributes (Task 2.8) provide far more operational value and were prioritized instead. + +--- + +## Task 2.6: Build Verification and Performance Baseline + +**Objective**: Verify the build succeeds with and without telemetry, and establish a performance baseline. + +**What to do**: + +1. Build with `telemetry=ON` and verify no compilation errors +2. Build with `telemetry=OFF` and verify no regressions +3. Run existing unit tests to verify no breakage +4. Document any build issues in lessons.md + +**Verification Checklist**: + +- [ ] `conan install . --build=missing -o telemetry=True` succeeds +- [ ] `cmake --preset default -Dtelemetry=ON` configures correctly +- [ ] Build succeeds with telemetry ON +- [ ] Build succeeds with telemetry OFF +- [ ] Existing tests pass with telemetry ON +- [ ] Existing tests pass with telemetry OFF + +--- + +## Task 2.8: RPC Span Attribute Enrichment — Node Health Context + +**Status**: DROPPED. + +Node health (`amendment_blocked`, `server_state`) is not part of the telemetry surface. Operators consume the same data via the existing `server_info` / `server_state` RPC commands, so duplicating it on traces adds storage and cardinality cost without new value. The OTel C++ SDK 1.18.0 also does not support runtime updates to the resource, ruling out resource-level emission of these dynamic-by-nature flags. + +--- + +## Task 2.9: PathFind RPC Instrumentation + +**Status**: COMPLETE + +**Objective**: Trace the path_find and ripple_path_find RPC handlers to capture request latency and computation cost. + +**Spans added**: + +- `pathfind.request` — wraps `doPathFind()` and `doRipplePathFind()` RPC handlers +- `pathfind.compute` — wraps `PathRequest::doUpdate()` (`pathfind_fast` attr) +- `pathfind.update_all` — wraps `PathRequestManager::updateAll()` on ledger close (`pathfind_ledger_index`, `pathfind_num_requests` attrs; emitted only when active subscriptions exist) +- `pathfind.discover` — wraps the entire per-source-asset loop in `PathRequest::findPaths()` (`pathfind_search_level`, `pathfind_num_paths` attrs). One span per RPC call instead of N (one per source asset). Trade-off: per-asset breakdown is lost; storage and cardinality bounded. + +**Attribute namespacing**: All pathfind attributes use the `pathfind_*` underscore form per the Phase 1c naming-spec rule 5. + +**New file**: `src/xrpld/rpc/detail/PathFindSpanNames.h` + +**Modified files**: + +- `src/xrpld/rpc/handlers/orderbook/PathFind.cpp` +- `src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp` +- `src/xrpld/rpc/detail/PathRequest.cpp` +- `src/xrpld/rpc/detail/PathRequestManager.cpp` +- `src/xrpld/rpc/detail/Pathfinder.cpp` + +--- + +## Task 2.10: RPC and PathFind Span Attribute Gap Fill + +**Status**: COMPLETE + +**Objective**: Wire up workflow-identifying attributes that enable filtering and grouping traces by request characteristics without drilling into child spans. + +**Attributes added**: + +| Span | Attribute | Type | Source | +| ------------------- | ---------------------------- | ------ | --------------------------------- | +| `rpc.http_request` | `request_payload_size` | int64 | `request.body().size()` | +| `rpc.process` | `is_batch` | bool | `method == "batch"` check | +| `rpc.process` | `batch_size` | int64 | `params.size()` (only when batch) | +| `rpc.ws_message` | `command` | string | `jv[command]` or `jv[method]` | +| `rpc.command.*` | `load_type` | string | `context.loadType.label()` | +| `pathfind.compute` | `pathfind_dest_currency` | string | `to_string(saDstAmount_.asset())` | +| `pathfind.discover` | `pathfind_num_source_assets` | int64 | `sourceAssets.size()` | + +_Note: `pathfind_dest_amount` was removed — the destination amount is a financial value excluded by the privacy policy (design §2.4.4)._ + +**New attr keys**: `RpcSpanNames.h` (`isBatch`, `batchSize`, `loadType`), `PathFindSpanNames.h` (`destCurrency`, `numSourceAssets`). + +**Modified files**: + +- `src/xrpld/rpc/detail/RpcSpanNames.h` +- `src/xrpld/rpc/detail/PathFindSpanNames.h` +- `src/xrpld/rpc/detail/ServerHandler.cpp` +- `src/xrpld/rpc/detail/RPCHandler.cpp` +- `src/xrpld/rpc/detail/PathRequest.cpp` + +--- + +## Summary + +| Task | Description | Status | Notes | +| ---- | ------------------------------------------- | ------------------- | --------------------------------------------------------- | +| 2.1 | W3C Trace Context header extraction | Deferred → Phase 3 | No consumer in Phase 2; needs cross-node tracing | +| 2.2 | Per-category span creation | Complete (Phase 1c) | Superseded by TraceCategory enum + SpanGuard | +| 2.3 | Add shouldTraceLedger() interface method | Complete (Phase 1c) | Delivered in Phase 1c base branch | +| 2.4 | Unit tests for core telemetry | Complete | TelemetryConfig + SpanGuardFactory tests | +| 2.5 | Enhanced RPC span attributes (HTTP-level) | Deferred | Low value; span duration covers timing natively | +| 2.6 | Build verification and performance baseline | Complete | Verified in CI on Phase 1c | +| 2.7 | Grafana Tempo search filters | Complete | rpc-command, rpc-status, rpc-role filters | +| 2.8 | RPC span attribute enrichment (node health) | Dropped | Available via `server_info`/`server_state` RPC | +| 2.9 | PathFind RPC instrumentation | Complete | request, compute, update_all, discover | +| 2.10 | RPC/PathFind span attribute gap fill | Complete | Batch detection, payload size, load cost, pathfind params | + +**Delivered in this branch**: Tasks 2.4, 2.7, 2.9, 2.10. +**Deferred with rationale**: Tasks 2.1 (→Phase 3), 2.5 (low priority). +**Dropped**: Task 2.8 (node health not duplicated on traces). +**Superseded**: Task 2.2 (Phase 1c SpanGuard factory covers this). + +--- + +## Known Issues / Future Work + +### Thread safety of TelemetryImpl::stop() vs startSpan() + +`TelemetryImpl::stop()` resets `sdkProvider_` (a `std::shared_ptr`) without +synchronization. `getTracer()` reads the same member from RPC handler threads. +This is a data race if any thread calls `startSpan()` concurrently with `stop()`. + +**Current mitigation**: `Application::stop()` shuts down `serverHandler_`, +`overlay_`, and `jobQueue_` before calling `telemetry_->stop()`, so no callers +remain. See comments in `Telemetry.cpp:stop()` and `Application.cpp`. + +**TODO**: Add an `std::atomic stopped_` flag checked in `getTracer()` to +make this robust against future shutdown order changes. + +### Macro incompatibility: XRPL_TRACE_SPAN vs XRPL_TRACE_SET_ATTR + +`XRPL_TRACE_SPAN` and `XRPL_TRACE_SPAN_KIND` declare `_xrpl_guard_` as a bare +`SpanGuard`, but `XRPL_TRACE_SET_ATTR` and `XRPL_TRACE_EXCEPTION` call +`_xrpl_guard_.has_value()` which requires `std::optional`. Using +`XRPL_TRACE_SPAN` followed by `XRPL_TRACE_SET_ATTR` in the same scope would +fail to compile. + +**Current mitigation**: No call site currently uses `XRPL_TRACE_SPAN` — all +production code uses the conditional macros (`XRPL_TRACE_RPC`, `XRPL_TRACE_TX`, +etc.) which correctly wrap the guard in `std::optional`. + +**TODO**: Either make `XRPL_TRACE_SPAN`/`XRPL_TRACE_SPAN_KIND` also wrap in +`std::optional`, or document that `XRPL_TRACE_SET_ATTR` is only compatible with +the conditional macros. diff --git a/OpenTelemetryPlan/Phase3_taskList.md b/OpenTelemetryPlan/Phase3_taskList.md new file mode 100644 index 00000000000..c5d3c95251b --- /dev/null +++ b/OpenTelemetryPlan/Phase3_taskList.md @@ -0,0 +1,555 @@ +# Phase 3: Transaction Tracing Task List + +> **Goal**: Trace the full transaction lifecycle from RPC submission through peer relay, including cross-node context propagation via Protocol Buffer extensions. This is the WALK phase that demonstrates true distributed tracing. +> +> **Scope**: Protocol Buffer `TraceContext` message, context serialization, PeerImp transaction instrumentation, NetworkOPs processing instrumentation, HashRouter visibility, and multi-node relay context propagation. +> +> **Branch**: `pratik/otel-phase3-tx-tracing` (from `pratik/otel-phase2-rpc-tracing`) + +### Related Plan Documents + +| Document | Relevance | +| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | +| [04-code-samples.md](./04-code-samples.md) | TraceContext protobuf (§4.4.1), PeerImp instrumentation (§4.5.1), context serialization (§4.4.2) | +| [01-architecture-analysis.md](./01-architecture-analysis.md) | Transaction flow (§1.3), key trace points (§1.6) | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 3 tasks (§6.4), definition of done (§6.11.3) | +| [02-design-decisions.md](./02-design-decisions.md) | Context propagation design (§2.5), attribute schema (§2.4.3) | + +--- + +## Task 3.1: Define TraceContext Protocol Buffer Message + +**Objective**: Add trace context fields to the P2P protocol messages so trace IDs can propagate across nodes. + +**What to do**: + +- Edit `include/xrpl/proto/xrpl.proto` (or `src/xrpld/proto/ripple.proto`, wherever the proto is): + - Add `TraceContext` message definition: + ```protobuf + message TraceContext { + bytes trace_id = 1; // 16-byte trace identifier + bytes span_id = 2; // 8-byte span identifier + uint32 trace_flags = 3; // bit 0 = sampled + string trace_state = 4; // W3C tracestate value + } + ``` + - Add `optional TraceContext trace_context = 1001;` to: + - `TMTransaction` + - `TMProposeSet` (for Phase 4 use) + - `TMValidation` (for Phase 4 use) + - Use high field numbers (1001+) to avoid conflicts with existing fields + +- Regenerate protobuf C++ code + +**Key modified files**: + +- `include/xrpl/proto/xrpl.proto` (or equivalent) + +**Reference**: + +- [04-code-samples.md §4.4.1](./04-code-samples.md) — TraceContext message definition +- [02-design-decisions.md §2.5.2](./02-design-decisions.md) — Protocol buffer context propagation design + +--- + +## Task 3.2: Implement Protobuf Context Serialization + +**Objective**: Create utilities to serialize/deserialize OTel trace context to/from protobuf `TraceContext` messages. + +**What to do**: + +- Create `include/xrpl/telemetry/TraceContextPropagator.h` (extend from Phase 2 if exists, or add protobuf methods): + - Add protobuf-specific methods: + - `static Context extractFromProtobuf(protocol::TraceContext const& proto)` — reconstruct OTel context from protobuf fields + - `static void injectToProtobuf(Context const& ctx, protocol::TraceContext& proto)` — serialize current span context into protobuf fields + - Both methods guard behind `#ifdef XRPL_ENABLE_TELEMETRY` + +- Create/extend `src/libxrpl/telemetry/TraceContextPropagator.cpp`: + - Implement extraction: read trace_id (16 bytes), span_id (8 bytes), trace_flags from protobuf, construct `SpanContext`, wrap in `Context` + - Implement injection: get current span from context, serialize its TraceId, SpanId, and TraceFlags into protobuf fields + +**Key new/modified files**: + +- `include/xrpl/telemetry/TraceContextPropagator.h` +- `src/libxrpl/telemetry/TraceContextPropagator.cpp` + +**Reference**: + +- [04-code-samples.md §4.4.2](./04-code-samples.md) — Full extract/inject implementation + +--- + +## Task 3.3: Instrument PeerImp Transaction Handling + +**Objective**: Add trace spans to the peer-level transaction receive and relay path. + +**What to do**: + +- Edit `src/xrpld/overlay/detail/PeerImp.cpp`: + - In `onMessage(TMTransaction)` / `handleTransaction()`: + - Extract parent trace context from incoming `TMTransaction::trace_context` field (if present) + - Create `tx.receive` span as child of extracted context (or new root if none) + - Set attributes: `tx_hash`, `peer_id`, `tx_status` + - On HashRouter suppression (duplicate): set `suppressed=true`, add `tx.duplicate` event + - Wrap validation call with child span `tx.validate` + - Wrap relay with `tx.relay` span + - When relaying to peers: + - Inject current trace context into outgoing `TMTransaction::trace_context` + - Set `relay_count` attribute + +- Use `SpanGuard::span(TraceCategory::Transactions, "tx", "receive")` factory + (Phase 1c replaced macros with the SpanGuard factory pattern) + +> **Note**: The `tx.receive` guard is `.detached()` before being moved into the +> `RcvCheckTx` job so its Scope is popped on the peer thread, not leaked to the +> worker (else later peer messages would inherit this transaction's trace). + +**Key modified files**: + +- `src/xrpld/overlay/detail/PeerImp.cpp` + +**Reference**: + +- [04-code-samples.md §4.5.1](./04-code-samples.md) — Full PeerImp instrumentation example +- [01-architecture-analysis.md §1.3](./01-architecture-analysis.md) — Transaction flow diagram +- [01-architecture-analysis.md §1.6](./01-architecture-analysis.md) — tx.receive trace point + +--- + +## Task 3.4: Instrument NetworkOPs Transaction Processing + +**Objective**: Trace the transaction processing pipeline in NetworkOPs, covering both sync and async paths. + +**What to do**: + +- Edit `src/xrpld/app/misc/NetworkOPs.cpp`: + - In `processTransaction()`: + - Create `tx.process` span + - Set attributes: `tx_hash`, `tx_type`, `local` (whether from RPC or peer) + - Record whether sync or async path is taken + - `.detached()` the guard before storing it in `TransactionStatus::span`, + since it is applied on a batch worker thread — this pops the Scope on the + origin thread and stops later work inheriting this transaction's trace + + - In `doTransactionAsync()`: + - Capture parent context before queuing + - Create `tx.queue` span with queue depth attribute + - Add event when transaction is dequeued for processing + + - In `doTransactionSync()`: + - Create `tx.process_sync` span + - Record result (applied, queued, rejected) + +**Key modified files**: + +- `src/xrpld/app/misc/NetworkOPs.cpp` + +**Reference**: + +- [01-architecture-analysis.md §1.6](./01-architecture-analysis.md) — tx.validate and tx.process trace points +- [02-design-decisions.md §2.4.3](./02-design-decisions.md) — Transaction attribute schema + +--- + +## Task 3.5: Instrument HashRouter for Dedup Visibility + +**Objective**: Make transaction deduplication visible in traces by recording HashRouter decisions as span attributes/events. + +**What to do**: + +- Edit `src/xrpld/overlay/detail/PeerImp.cpp` (in handleTransaction): + - After calling `HashRouter::shouldProcess()` or `addSuppressionPeer()`: + - Record `suppressed` attribute (true/false) + - Record `tx_flags` showing current HashRouter state (SAVED, TRUSTED, etc.) + - Add `tx.first_seen` or `tx.duplicate` event + +- This is NOT a modification to HashRouter itself — just recording its decisions as span attributes in the existing PeerImp instrumentation from Task 3.3. + +**Key modified files**: + +- `src/xrpld/overlay/detail/PeerImp.cpp` (same changes as 3.3, logically grouped) + +--- + +## Task 3.6: Context Propagation in Transaction Relay + +**Status**: COMPLETE (transaction relay). Consensus proposal/validation +propagation is deferred to Phase 4 — see "Planned (Phase 4)" below. + +**Objective**: Ensure trace context flows correctly when transactions are relayed between peers, creating linked spans across nodes. + +**What was done**: + +- **TX send side**: `NetworkOPs::apply()` now injects the tx.process span's trace + context into the outgoing `TMTransaction` protobuf before relay, using + `telemetry::injectSpanContext()`. The receiving node's `txReceiveSpan()` (already + wired in PeerImp) extracts the parent span_id and creates the tx.receive span + as a child of the sender's tx.process span. + +- **Edge cases**: Missing trace context (older peers) degrades gracefully to + standalone spans. Invalid/corrupted context is treated as absent. Trace + flags are propagated and respected. + +**New infrastructure**: + +- `SpanGuard::getTraceBytes()` — extracts raw trace_id/span_id/trace_flags + from a span without exposing OTel types. Safe to call from any thread. +- `PropagationHelpers.h` — `injectSpanContext(SpanGuard&, proto)` bridge + between SpanGuard and protobuf TraceContext. +- `TraceContextPropagator.h` — `injectToProtobuf(ctx, proto)` for + same-thread injection via OTel RuntimeContext. + +**Key modified files**: + +- `src/xrpld/app/misc/NetworkOPs.cpp` — tx relay injection +- `include/xrpl/telemetry/SpanGuard.h` — `TraceBytes` struct, `getTraceBytes()` +- `src/libxrpl/telemetry/SpanGuard.cpp` — `getTraceBytes()` implementation +- `src/xrpld/telemetry/PropagationHelpers.h` — inject helpers (new file) + +**Planned (Phase 4 — not in this PR)**: + +The consensus proposal/validation propagation below is Phase 4 scope and is +not implemented on this branch. It is listed here only to record the intended +design. + +- **Proposal send/receive**: `RCLConsensus::Adaptor::propose()` injects the + current thread's active span context into the `TMProposeSet` protobuf via + `telemetry::injectToProtobuf()`. PeerImp creates a + `consensus.proposal.receive` span that extracts the sender's trace context + as parent (via `ConsensusReceiveTracing.h`). + +- **Validation send/receive**: `RCLConsensus::Adaptor::validate()` injects + the current thread's active span context into the `TMValidation` protobuf. + PeerImp creates a `consensus.validation.receive` span that extracts the + sender's trace context as parent. + +- Planned files: `src/xrpld/app/consensus/RCLConsensus.cpp` (send injection), + `src/xrpld/overlay/detail/PeerImp.cpp` (receive spans), + `src/xrpld/telemetry/ConsensusReceiveTracing.h` (receive span helpers, + new file). + +**Reference**: + +- [02-design-decisions.md §2.5](./02-design-decisions.md) — Context propagation design +- [04-code-samples.md §4.5.1](./04-code-samples.md) — Relay context injection pattern + +--- + +## Task 3.7: Build Verification and Testing + +**Objective**: Verify all Phase 3 changes compile and work correctly. + +**What to do**: + +1. Build with `telemetry=ON` — verify no compilation errors +2. Build with `telemetry=OFF` — verify no regressions +3. Run existing unit tests +4. Verify protobuf regeneration produces correct C++ code +5. Document any issues encountered + +**Verification Checklist**: + +- [ ] Protobuf changes generate valid C++ +- [ ] Build succeeds with telemetry ON +- [ ] Build succeeds with telemetry OFF +- [ ] Existing tests pass +- [ ] No undefined symbols from new telemetry calls + +--- + +## Task 3.8: Transaction Span Peer Version Attribute + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — adds peer version context inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). +> +> **Upstream**: Phase 2 (RPC span infrastructure must exist). +> **Downstream**: Phase 10 (validation checks for this attribute). + +**Objective**: Add the relaying peer's xrpld version to `tx.receive` spans so operators can correlate transaction issues with peer version mismatches during network upgrades. + +**What to do**: + +- Edit `src/xrpld/overlay/detail/PeerImp.cpp`: + - In the `tx.receive` span block (after existing `peer_id` setAttribute call): + - Add `peer_version` (string) — from `this->getVersion()` + - Only set if `getVersion()` returns a non-empty string (avoid empty-string attributes) + +**New span attribute**: + +| Attribute | Type | Source | Example | +| -------------- | ------ | -------------------- | --------------- | +| `peer_version` | string | `peer->getVersion()` | `"xrpld-2.4.0"` | + +**Rationale**: Transaction relay is where version mismatches cause subtle serialization or validation bugs. Tracing "this tx came from a v2.3.0 peer" helps diagnose compatibility issues. The community dashboard tracks peer versions externally; this brings version awareness into the trace itself. + +**Key modified files**: + +- `src/xrpld/overlay/detail/PeerImp.cpp` + +**Exit Criteria**: + +- [ ] `tx.receive` spans carry `peer_version` attribute with a non-empty version string +- [ ] Attribute is omitted (not set to empty string) when `getVersion()` returns empty +- [ ] Attribute visible in Tempo trace detail view + +--- + +## Task 3.9: Deterministic Transaction Trace ID + +> **Upstream**: Task 3.2 (protobuf serialization), Task 3.3 (PeerImp span exists). +> **Downstream**: Phase 10 (workload validation can query by tx hash directly). +> **Pattern**: Mirrors the consensus deterministic trace ID in Phase 4a +> (`createDeterministicContext` in `RCLConsensus.cpp`), adapted for transactions. + +**Objective**: Derive the trace_id for transaction spans deterministically from the +transaction hash so that all nodes handling the same transaction independently produce +spans under the same trace_id — regardless of whether protobuf context propagation +succeeds. + +**Why**: The current approach creates spans with random trace_ids and relies entirely +on protobuf `TraceContext` propagation to link them. If any hop in the relay chain +drops the context (older peers, message corruption, mixed-version networks), the trace +splits and downstream spans become impossible to find. With deterministic trace_ids, +correlation is guaranteed because every node derives the same trace_id from the same +`txID`. + +**Approach — deterministic trace_id + protobuf span_id propagation**: + +1. Derive `trace_id = txHash[0:16]` (first 16 bytes of the 32-byte transaction hash). +2. Generate a random 8-byte `span_id` per node (each node's span is unique within + the shared trace). +3. Create the span under this deterministic context as parent. +4. **Additionally**, if protobuf `TraceContext` is present in the incoming + `TMTransaction` message, extract the sender's `span_id` and use it as the span's + parent — this preserves parent-child ordering in the trace tree. +5. If protobuf context is absent (older peer, first hop), the span still has the + correct deterministic `trace_id` — it appears as a sibling root in the same trace + rather than being lost. + +This gives the best of both worlds: guaranteed cross-node correlation via deterministic +`trace_id`, plus parent-child relay ordering via protobuf `span_id` when available. + +**What to do**: + +- Create `createDeterministicTxContext(uint256 const& txHash)` utility function: + - Location: shared header or file-local in `PeerImp.cpp` and `NetworkOPs.cpp` + (or a shared telemetry utility if both need it). + - Pattern: identical to `createDeterministicContext(uint256 const& ledgerId)` in + `RCLConsensus.cpp` — take `txHash[0:16]` as trace_id, random span_id via + `default_prng()`, sampled flag set, `remote=false`. + - Guard behind `#ifdef XRPL_ENABLE_TELEMETRY`. + + ```cpp + opentelemetry::context::Context + createDeterministicTxContext(uint256 const& txHash) + { + namespace trace = opentelemetry::trace; + + // First 16 bytes of the 32-byte tx hash as trace ID. + trace::TraceId traceId( + opentelemetry::nostd::span(txHash.data(), 16)); + + // Random span_id so each node's span is unique within the trace. + uint8_t spanIdBytes[8]; + auto const rval = default_prng()(); + std::memcpy(spanIdBytes, &rval, sizeof(spanIdBytes)); + trace::SpanId spanId( + opentelemetry::nostd::span(spanIdBytes, 8)); + + trace::SpanContext syntheticCtx( + traceId, spanId, trace::TraceFlags(1), /* remote = */ false); + + return opentelemetry::context::Context{}.SetValue( + trace::kSpanKey, + opentelemetry::nostd::shared_ptr( + new trace::DefaultSpan(syntheticCtx))); + } + ``` + +- Edit `src/xrpld/overlay/detail/PeerImp.cpp` — restructure `handleTransaction()`: + - **Move span creation after deserialization** (txID must be known first): + 1. Deserialize `STTx` and get `txID` (existing code at line ~1382). + 2. Create deterministic parent context: `auto detCtx = createDeterministicTxContext(txID)`. + 3. If `m->has_trace_context()`: extract protobuf context via `extractFromProtobuf()`, + **combine** with deterministic trace_id — use the protobuf span_id as parent + to preserve relay ordering, but override trace_id with the deterministic one. + 4. If no protobuf context: create span under `detCtx` directly. + 5. Set all existing attributes (`hash`, `peerId`, `peerVersion`, `suppressed`, etc.). + + - **Combining deterministic trace_id with protobuf parent span_id**: + When both are available, construct a synthetic `SpanContext` with: + - `trace_id` = `txHash[0:16]` (deterministic) + - `span_id` = extracted from protobuf (sender's span_id → becomes parent) + - `trace_flags` = from protobuf + - `remote` = true (came from another node) + + ```cpp + // Pseudo-code for the combined context: + auto detTraceId = trace::TraceId(txHash.data(), 16); + auto remoteSpanId = /* from extractFromProtobuf */; + auto remoteFlags = /* from extractFromProtobuf */; + + trace::SpanContext combinedCtx( + detTraceId, remoteSpanId, remoteFlags, /* remote = */ true); + // Use as parent context for the new span. + ``` + +- Edit `src/xrpld/app/misc/NetworkOPs.cpp` — update `processTransaction()`: + - `transaction->getID()` is already available at the top of the function. + - Create deterministic parent context from `txID`. + - Create `tx.process` span under this context. + - No protobuf context to extract here (NetworkOPs is intra-node), so + deterministic context alone is sufficient. + +- Add `trace_strategy` attribute to spans: + - Add `inline constexpr auto traceStrategy = "trace_strategy";` + to `TxSpanNames.h`. + - Set on each tx span: `span.setAttribute(tx_span::attr::traceStrategy, "deterministic")`. + +**Key new/modified files**: + +- `src/xrpld/overlay/detail/PeerImp.cpp` — restructured span creation +- `src/xrpld/app/misc/NetworkOPs.cpp` — deterministic context for tx.process +- `src/xrpld/telemetry/TxSpanNames.h` — new `traceStrategy` attribute constant +- New or shared utility for `createDeterministicTxContext()` (location TBD: could be + a shared header like `include/xrpl/telemetry/DeterministicContext.h`, or file-local + if only used in two places) + +**Interaction with existing tasks**: + +- **Task 3.3 (PeerImp instrumentation)**: The span creation in `handleTransaction()` + must be restructured — the span currently starts before `txID` is known. This task + moves it after deserialization. +- **Task 3.6 (Relay context propagation)**: Protobuf injection at the relay site + remains the same — `injectToProtobuf()` serializes the current span's `span_id`. + The receiver extracts it and combines with the deterministic `trace_id`. +- **Phase 4a (Consensus deterministic trace ID)**: This task follows the same pattern. + Consider extracting a shared utility (e.g., `createDeterministicContext(uint256)`) + that both consensus and transaction tracing use. + +**Exit Criteria**: + +- [ ] `tx.receive` and `tx.process` spans have deterministic trace_id = `txHash[0:16]` +- [ ] All nodes handling the same transaction produce spans under the same trace_id +- [x] Protobuf `span_id` propagation still works when available (parent-child ordering) +- [ ] Missing protobuf context (old peer) degrades gracefully to sibling spans, not lost traces +- [ ] `trace_strategy` attribute set to `"deterministic"` on all tx spans +- [ ] Trace queryable by tx hash (truncate hash → trace_id → direct lookup in Tempo) + +**Deliverables implemented (not in original plan)**: + +- **`SpanGuard::txSpan()` factory method** (`include/xrpl/telemetry/SpanGuard.h`): + Two overloads for creating transaction spans with deterministic trace IDs: + - `txSpan(category, group, name, txHash)` — standalone span (deterministic + trace_id from `txHash[0:16]`, no parent span_id). + - `txSpan(category, group, name, txHash, parentCtx)` — child span (deterministic + trace_id combined with protobuf-extracted parent span_id for relay ordering). + +- **`TxTracing.h` helper functions** (`src/xrpld/telemetry/TxTracing.h`): + File-local helpers that wrap `SpanGuard::txSpan()` for the two main PeerImp call + sites: + - `txReceiveSpan(txHash, parentCtx)` — creates `tx.receive` span with + deterministic trace_id and optional protobuf parent context. + - `txProcessSpan(txHash)` — creates `tx.process` span with deterministic + trace_id only (no protobuf parent, used intra-node). + - **Note**: `TxTracing.h` includes `xrpl.pb.h` unconditionally (outside + `#ifdef XRPL_ENABLE_TELEMETRY`) because `protocol::TMTransaction` appears in + the function signatures regardless of telemetry build mode. + +--- + +## Task 3.10: TxQ Instrumentation + +**Status**: COMPLETE + +**Objective**: Trace the transaction queue lifecycle — enqueue decisions, direct apply, batch clear, ledger-close accept loop, per-tx apply, and cleanup. + +**Spans added**: + +- `txq.enqueue` — wraps `TxQ::apply()` with tx_hash attribute +- `txq.apply_direct` — wraps `TxQ::tryDirectApply()` fast-path +- `txq.batch_clear` — wraps `TxQ::tryClearAccountQueueUpThruTx()` +- `txq.accept` — wraps `TxQ::accept()` ledger-close dequeue with queue_size attr +- `txq.accept_tx` — per-tx span inside accept loop with tx_hash, ter_code, + retries_remaining attributes +- `txq.cleanup` — wraps `TxQ::processClosedLedger()` with ledger_seq attribute + +**New file**: `src/xrpld/app/misc/detail/TxQSpanNames.h` + +**Modified file**: `src/xrpld/app/misc/detail/TxQ.cpp` + +--- + +## Task 3.11: TX and TxQ Span Attribute Gap Fill + +**Status**: COMPLETE + +**Objective**: Add workflow-identifying attributes to transaction spans so operators can filter by transaction type and see outcomes without off-chain correlation. + +**Attributes added**: + +| Span | Attribute | Type | Source | +| ----------------- | -------------------- | ------ | ------------------------------------------------------------------- | +| `tx.process` | `tx_type` | string | `TxFormats::getInstance().findByType(stx->getTxnType())->getName()` | +| `tx.process` | `fee` | int64 | `stx->getFieldAmount(sfFee).xrp().drops()` | +| `tx.process` | `sequence` | int64 | `stx->getSeqProxy().value()` | +| `tx.process` | `ter_result` | string | `transToken(e.result)` (set after batch application) | +| `tx.process` | `applied` | bool | `e.applied` (set after batch application) | +| `tx.receive` | `tx_type` | string | `TxFormats::getInstance().findByType(stx->getTxnType())->getName()` | +| `txq.enqueue` | `tx_type` | string | same pattern as above | +| `txq.enqueue` | `txq_status` | string | `queued` / `applied_direct` / `applied` / `rejected` | +| `txq.enqueue` | `fee_level_paid` | int64 | `getFeeLevelPaid(view, *tx).value()` | +| `txq.enqueue` | `required_fee_level` | int64 | `getRequiredFeeLevel(...).value()` | +| `txq.batch_clear` | `num_cleared` | int64 | queued txs cleared ahead of the applying tx | +| `txq.cleanup` | `expired_count` | int64 | entries dropped for passed `LastLedgerSequence` | +| `txq.accept_tx` | `txq_status` | string | `applied` / `failed` / `retried` | +| `txq.accept` | `ledger_changed` | bool | set at end of accept loop | + +**New attr keys**: `TxSpanNames.h` (`txType`, `fee`, `sequence`, `terResult`, `applied`), `TxQSpanNames.h` (`txType`). + +**Modified files**: + +- `src/xrpld/telemetry/TxSpanNames.h` +- `src/xrpld/app/misc/detail/TxQSpanNames.h` +- `src/xrpld/app/misc/NetworkOPs.cpp` +- `src/xrpld/overlay/detail/PeerImp.cpp` +- `src/xrpld/app/misc/detail/TxQ.cpp` + +--- + +## Summary + +| Task | Description | New Files | Modified Files | Depends On | +| ---- | ----------------------------------- | --------- | -------------- | ---------- | +| 3.1 | TraceContext protobuf message | 0 | 1 | Phase 2 | +| 3.2 | Protobuf context serialization | 1-2 | 0 | 3.1 | +| 3.3 | PeerImp transaction instrumentation | 0 | 1 | 3.2 | +| 3.4 | NetworkOPs transaction processing | 0 | 1 | Phase 2 | +| 3.5 | HashRouter dedup visibility | 0 | 1 | 3.3 | +| 3.6 | Relay context propagation | 0 | 1-2 | 3.3, 3.5 | +| 3.7 | Build verification and testing | 0 | 0 | 3.1-3.6 | +| 3.8 | TX span peer version attribute | 0 | 1 | 3.3 | +| 3.9 | Deterministic transaction trace ID | 0-1 | 3 | 3.2, 3.3 | +| 3.10 | TxQ instrumentation (6 spans) | 1 | 1 | 3.4 | +| 3.11 | TX/TxQ span attribute gap fill | 0 | 5 | 3.3, 3.10 | + +**Parallel work**: Tasks 3.1 and 3.4 can start in parallel. Task 3.2 depends on 3.1. Tasks 3.3 and 3.5 depend on 3.2. Task 3.6 depends on 3.3 and 3.5. Task 3.8 depends on 3.3 (span must exist). Task 3.9 depends on 3.2 and 3.3. Task 3.10 depends on 3.4 (tx.process span must exist). + +**Exit Criteria** (from [06-implementation-phases.md §6.11.3](./06-implementation-phases.md)): + +- [x] Transaction traces span across nodes +- [x] Trace context in Protocol Buffer messages +- [ ] HashRouter deduplication visible in traces +- [ ] <5% overhead on transaction throughput +- [x] Deterministic trace_id: same trace_id for same tx across all nodes +- [x] Protobuf span_id propagation preserves parent-child ordering when available + +--- + +## Known Issues / Future Work + +### Unused trace_state proto field + +The `TraceContext.trace_state` field (field 4) in `xrpl.proto` is reserved for +W3C `tracestate` vendor-specific key-value pairs but is not read or written by +`TraceContextPropagator`. Wire it when cross-vendor trace propagation is needed. +No wire cost since proto `optional` fields are zero-cost when absent. diff --git a/OpenTelemetryPlan/Phase4_taskList.md b/OpenTelemetryPlan/Phase4_taskList.md new file mode 100644 index 00000000000..e83a16262ef --- /dev/null +++ b/OpenTelemetryPlan/Phase4_taskList.md @@ -0,0 +1,946 @@ +# Phase 4: Consensus Tracing Task List + +> **Goal**: Full observability into consensus rounds — track round lifecycle, phase transitions, proposal handling, and validation. This is the RUN phase that completes the distributed tracing story. +> +> **Scope**: RCLConsensus instrumentation for round starts, phase transitions (open/establish/accept), proposal send/receive, validation handling, and correlation with transaction traces from Phase 3. +> +> **Branch**: `pratik/otel-phase4-consensus-tracing` (from `pratik/otel-phase3-tx-tracing`) + +> **Note on attribute names**: the `xrpl..` keys shown below are +> written in the older dotted form for readability — it mirrors how the fully +> qualified attribute reads in a Tempo trace view. The implemented keys follow +> the convention in [CONTRIBUTING.md](../CONTRIBUTING.md#telemetry-span-attribute-naming) +> (underscore form, e.g. `consensus_round`, `consensus_mode`); the +> `*SpanNames.h` constants are the single source of truth. + +### Related Plan Documents + +| Document | Relevance | +| ------------------------------------------------------------ | ----------------------------------------------------------- | +| [04-code-samples.md](./04-code-samples.md) | Consensus instrumentation (§4.5.2), consensus span patterns | +| [01-architecture-analysis.md](./01-architecture-analysis.md) | Consensus round flow (§1.4), key trace points (§1.6) | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 4 tasks (§6.5), definition of done (§6.11.4) | +| [02-design-decisions.md](./02-design-decisions.md) | Consensus attribute schema (§2.4.4) | + +--- + +## Task 4.1: Instrument Consensus Round Start ✅ + +**Objective**: Create a root span for each consensus round that captures the round's key parameters. + +**Status**: DONE (implemented via Task 4a.2 `startRoundTracing()` helper). + +**What was done**: + +- `RCLConsensus::Adaptor::startRoundTracing()` creates `consensus.round` span + via `SpanGuard::hashSpan()` (deterministic) or `SpanGuard::span()` (attribute strategy) +- Attributes set: `xrpl.consensus.ledger_id`, `xrpl.ledger.seq`, + `xrpl.consensus.mode`, `trace_strategy`, `xrpl.consensus.round_id` +- Round span stored as `roundSpan_` member in `RCLConsensus::Adaptor` +- `roundSpanContext_` snapshot captured for cross-thread span linking + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` +- `src/xrpld/app/consensus/RCLConsensus.h` (span and context members) + +**Reference**: + +- [04-code-samples.md §4.5.2](./04-code-samples.md) — startRound instrumentation example +- [01-architecture-analysis.md §1.4](./01-architecture-analysis.md) — Consensus round flow + +--- + +## Task 4.2: Instrument Phase Transitions ✅ + +**Objective**: Create child spans for each consensus phase (open, establish, accept) to show timing breakdown. + +**Status**: DONE. All consensus phases are now instrumented: + +- `consensus.establish` — created in `Consensus.h::startEstablishTracing()` +- `consensus.ledger_close` — created in `RCLConsensus.cpp::onClose()` +- `consensus.accept` / `consensus.accept.apply` — created in `onAccept()` / `doAccept()` +- `consensus.phase.open` — `openSpan_` member in `Consensus.h`, created in `startRoundInternal()`, ended in `closeLedger()` + +**Design notes**: + +- `phase` attribute — phases are distinguished by span names instead +- `phase.enter` / `phase.exit` events — not added (span start/end serves this purpose) +- `phase_duration_ms` attribute — not set (span duration captures this) + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` +- `src/xrpld/consensus/Consensus.h` (template-level establish phase tracking) + +**Reference**: + +- [04-code-samples.md §4.5.2](./04-code-samples.md) — phaseTransition instrumentation + +--- + +## Task 4.3: Instrument Proposal Handling ✅ + +**Objective**: Trace proposal send and receive to show validator coordination. + +**Status**: DONE. Both send and receive paths are instrumented. + +**What was done**: + +- In `Adaptor::propose()`: + - Creates `consensus.proposal.send` span via `SpanGuard::span()` + - Sets `xrpl.consensus.round` attribute + +- In `PeerImp::onMessage(TMProposeSet)`: + - Creates `consensus.proposal.receive` span + - Sets `trusted` attribute (bool) + +**Done here** (cross-node propagation, send + receive): + +- Trace context injection for `TMProposeSet::trace_context` in `propose()` +- Receive-side extraction in `PeerImp::onMessage(TMProposeSet)` via + `telemetry::proposalReceiveSpan()` (parents the receive span on the + sender's context when a valid `trace_context` is present) + +**Not implemented** (deferred to Phase 4b): + +- `consensus.proposal.relay` span in `share(RCLCxPeerPos)` + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` + +**Reference**: + +- [04-code-samples.md §4.5.2](./04-code-samples.md) — peerProposal instrumentation +- [02-design-decisions.md §2.4.4](./02-design-decisions.md) — Consensus attribute schema + +--- + +## Task 4.4: Instrument Validation Handling ✅ + +**Objective**: Trace validation send and receive to show ledger validation flow. + +**Status**: DONE. Both send and receive paths are instrumented. + +**What was done**: + +- In `Adaptor::validate()` (called from `doAccept()`): + - Creates `consensus.validation.send` span via `Adaptor::createValidationSpan()` + - Uses `SpanGuard::linkedSpan()` to create a follows-from link to the round span + - Thread-safe: uses `roundSpanContext_` snapshot (captured on consensus thread, + read on jtACCEPT thread) + - Sets `xrpl.ledger.seq` and `proposing` attributes + +- In `PeerImp::onMessage(TMValidation)`: + - Creates `consensus.validation.receive` span + - Sets `trusted` attribute (bool) + - Sets `xrpl.ledger.seq` attribute + +**Not implemented** (deferred to Phase 4b — cross-node propagation): + +- Validated ledger hash, signing time attributes on send span (see Task 4.8) + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` + +--- + +## Task 4.5: Add Consensus-Specific Attributes ✅ + +**Objective**: Enrich consensus spans with detailed attributes for debugging and analysis. + +**Status**: DONE. All core attributes are set across various spans, including the previously missing `tx_count` and `disputes_count`. + +**Implemented attributes** (across various spans): + +- `xrpl.ledger.seq` — on `consensus.round`, `consensus.accept.apply` +- `xrpl.consensus.round` — on `consensus.proposal.send` +- `xrpl.consensus.mode` — on `consensus.round`, `consensus.ledger_close` +- `proposers` — on `consensus.accept`, `consensus.establish`, `consensus.update_positions` +- `converge_percent` — on `consensus.establish`, `consensus.update_positions`, `consensus.check` +- `tx_count` — on `consensus.accept.apply` span (in `doAccept()`) +- `disputes_count` — on `consensus.update_positions` span (in `updateOurPositions()`) + +**Design notes**: + +- `phase` — phases distinguished by span names instead +- `phase_duration_ms` — span duration captures this + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` +- `src/xrpld/consensus/Consensus.h` + +--- + +## Task 4.6: Correlate Transaction and Consensus Traces ✅ + +**Objective**: Link transaction traces from Phase 3 with consensus traces so you can follow a transaction from submission through consensus into the ledger. + +**Status**: DONE. Transaction-consensus correlation implemented via `tx.included` events in `doAccept()`. + +**What was done**: + +- In `doAccept()` (RCLConsensus.cpp): + - Records `tx.included` events on the `consensus.accept.apply` span for each transaction in the accepted set + - Each event includes `xrpl.tx.id` attribute with the transaction hash + - This links consensus traces to individual transactions + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` + +--- + +## Task 4.7: Build Verification and Testing ✅ + +**Objective**: Verify all Phase 4 changes compile and don't affect consensus timing. + +**What to do**: + +1. Build with `telemetry=ON` — verify no compilation errors +2. Build with `telemetry=OFF` — verify no regressions (critical for consensus code) +3. Run existing consensus-related unit tests +4. Verify that `SpanGuard` factory methods compile to no-ops when disabled +5. Check that no consensus-critical code paths are affected by instrumentation overhead + +**Verification Checklist**: + +- [x] Build succeeds with telemetry ON +- [x] Build succeeds with telemetry OFF +- [x] Existing consensus tests pass +- [x] `SpanGuard` no-op implementation prevents overhead when telemetry is OFF +- [x] Phase timing instrumentation doesn't use blocking operations + +--- + +## Task 4.8: Consensus Validation Span Enrichment — NOT DONE + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — adds validation agreement context inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). +> +> **Upstream**: Phase 4 tasks 4.1-4.4 (span creation must exist). +> **Downstream**: Phase 7 (ValidationTracker reads these attributes), Phase 10 (validation checks). + +**Objective**: Add ledger hash, validation type, and quorum data to consensus validation spans on both send and receive paths. This enables trace-level validation agreement analysis — filter by ledger hash to see which validators agreed for a given ledger. + +**Status**: Not implemented. None of the enrichment attributes are set. The `consensus.validation.send` span only has `ledger.seq` and `proposing`. The `consensus.accept` span has `quorum` set to `result.proposers` (not the actual validator quorum from `app_.validators().quorum()`). No `PeerImp.cpp` changes were made. + +**What to do**: + +- Edit `src/xrpld/app/consensus/RCLConsensus.cpp`: + - On the `consensus.validation.send` span (in `validate()` / `doAccept()`): + - Add `xrpl.validation.ledger_hash` (string) — the ledger hash being validated + - Add `xrpl.validation.full` (bool) — whether this is a full validation (not partial) + - On the `consensus.accept` span (in `onAccept()`): + - Add `validation_quorum` (int64) — from `app_.validators().quorum()` + - Add `proposers_validated` (int64) — from `result.proposers` + +- Edit `src/xrpld/overlay/detail/PeerImp.cpp`: + - On the `peer.validation.receive` span: + - Add `xrpl.peer.validation.ledger_hash` (string) — from deserialized `STValidation` object + - Add `xrpl.peer.validation.full` (bool) — from `STValidation` flags + +**New span attributes**: + +| Span | Attribute | Type | Source | +| --------------------------- | ---------------------------------- | ------ | --------------------------------- | +| `consensus.validation.send` | `xrpl.validation.ledger_hash` | string | Ledger hash from validate() args | +| `consensus.validation.send` | `xrpl.validation.full` | bool | Full vs partial validation | +| `peer.validation.receive` | `xrpl.peer.validation.ledger_hash` | string | From STValidation deserialization | +| `peer.validation.receive` | `xrpl.peer.validation.full` | bool | From STValidation flags | +| `consensus.accept` | `validation_quorum` | int64 | `app_.validators().quorum()` | +| `consensus.accept` | `proposers_validated` | int64 | `result.proposers` | + +**Rationale**: The external dashboard's most valuable feature is validation agreement tracking. By recording the ledger hash on both outgoing and incoming validation spans, we create the raw data for agreement analysis at the trace level. Example Tempo query: + +``` +{name="consensus.validation.send"} | xrpl.validation.ledger_hash = "A1B2C3..." +``` + +Phase 7's `ValidationTracker` builds metric-level aggregation (1h/24h agreement %) on top of this data. + +**Key modified files (not yet modified)**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` +- `src/xrpld/overlay/detail/PeerImp.cpp` + +**Exit Criteria**: + +- [x] `consensus.validation.send` spans carry `ledger_hash` and `full_validation` +- [ ] `peer.validation.receive` spans carry `xrpl.peer.validation.ledger_hash` and `xrpl.peer.validation.full` +- [ ] `consensus.accept` spans carry `validation_quorum` and `proposers_validated` +- [x] Ledger hash attributes match between send and receive for the same ledger +- [ ] No impact on consensus performance + +--- + +## Task 4.9: Consensus Span Attribute Gap Fill + +**Status**: COMPLETE + +**Objective**: Add workflow-critical attributes to consensus spans that enable operators to understand consensus outcomes, identify bow-out proposals, and correlate validations to specific ledgers. + +**Attributes added**: + +| Span | Attribute | Type | Source | +| --------------------------- | ----------------- | ------ | ------------------------------------- | +| `consensus.proposal.send` | `is_bow_out` | bool | `proposal.isBowOut()` | +| `consensus.accept` | `consensus_state` | string | `result.state` (yes/moved_on/expired) | +| `consensus.accept` | `disputes_count` | int64 | `result.disputes.size()` | +| `consensus.validation.send` | `ledger_hash` | string | `ledger.ledger->header().hash` | + +**New attr keys**: `ConsensusSpanNames.h` (`isBowOut`, `ledgerHash`). + +**Modified files**: + +- `src/xrpld/consensus/ConsensusSpanNames.h` +- `src/xrpld/app/consensus/RCLConsensus.cpp` + +--- + +## Summary + +| Task | Description | Status | New Files | Modified Files | Depends On | +| ---- | ------------------------------------------- | ----------- | --------- | -------------- | ------------- | +| 4.1 | Consensus round start instrumentation | ✅ Done | 0 | 2 | Phase 3 | +| 4.2 | Phase transition instrumentation | ✅ Done | 0 | 1-2 | 4.1 | +| 4.3 | Proposal handling instrumentation | ✅ Done | 0 | 2 | 4.1 | +| 4.4 | Validation handling instrumentation | ✅ Done | 0 | 2 | 4.1 | +| 4.5 | Consensus-specific attributes | ✅ Done | 0 | 2 | 4.2, 4.3, 4.4 | +| 4.6 | Transaction-consensus correlation | ✅ Done | 0 | 1 | 4.2, Phase 3 | +| 4.7 | Build verification and testing | ✅ Done | 0 | 0 | 4.1-4.6 | +| 4.8 | Validation span enrichment (ext. dashboard) | ❌ Not done | 0 | 2 | 4.4 | +| 4.9 | Consensus span attribute gap fill | ✅ Done | 0 | 2 | 4.1-4.5 | + +**Parallel work**: Tasks 4.2, 4.3, and 4.4 can run in parallel after 4.1 is complete. Task 4.5 depends on all three. Task 4.6 depends on 4.2 and Phase 3. Task 4.8 depends on 4.4 (validation spans must exist). + +### Implemented Spans + +| Span Name | Method | Key Attributes | +| --------------------------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `consensus.proposal.send` | `Adaptor::propose` | `xrpl.consensus.round`, `is_bow_out` | +| `consensus.ledger_close` | `Adaptor::onClose` | `xrpl.ledger.seq`, `xrpl.consensus.mode` | +| `consensus.accept` | `Adaptor::onAccept` | `proposers`, `round_time_ms`, `quorum`, `disputes_count`, `consensus_state` | +| `consensus.accept.apply` | `Adaptor::doAccept` | `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `xrpl.ledger.seq`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction` | +| `consensus.validation.send` | `Adaptor::onAccept` (via validate) | `proposing`, `ledger_hash`, `ledger_seq`, `full_validation`, `validation_sign_time` | + +#### Close Time Attributes (consensus.accept.apply) + +The `consensus.accept.apply` span captures ledger close time agreement details +driven by `avCT_CONSENSUS_PCT` (75% validator agreement threshold): + +- **`close_time`** — Agreed-upon ledger close time (epoch seconds). When validators disagree (`consensusCloseTime == epoch`), this is synthetically set to `prevCloseTime + 1s`. +- **`close_time_correct`** — `true` if validators reached agreement, `false` if they "agreed to disagree" (close time forced to prev+1s). +- **`close_resolution_ms`** — Rounding granularity for close time (starts at 30s, decreases as ledger interval stabilizes). +- **`consensus_state`** — `"finished"` (normal) or `"moved_on"` (consensus failed, adopted best available). +- **`proposing`** — Whether this node was proposing. +- **`round_time_ms`** — Total consensus round duration. +- **`parent_close_time`** — Previous ledger's close time (epoch seconds). Enables computing close-time deltas across consecutive rounds without correlating separate spans. +- **`close_time_self`** — This node's own proposed close time before consensus voting. +- **`close_time_vote_bins`** — Number of distinct close-time vote bins from peer proposals. Higher values indicate less agreement among validators. +- **`resolution_direction`** — Whether close-time resolution `"increased"` (coarser), `"decreased"` (finer), or stayed `"unchanged"` relative to the previous ledger. + +**Exit Criteria** (from [06-implementation-phases.md §6.11.4](./06-implementation-phases.md)): + +- [x] Complete consensus round traces +- [x] Phase transitions visible (open, establish, close, accept) +- [x] Proposals and validations traced — send and receive; relay deferred to Phase 4b +- [x] Close time agreement tracked (per `avCT_CONSENSUS_PCT`) +- [x] No impact on consensus timing +- [x] Transaction-consensus correlation (Task 4.6) — `tx.included` events in doAccept +- [ ] Validation span enrichment (Task 4.8) — not implemented + +--- + +# Phase 4a: Establish-Phase Gap Fill & Cross-Node Correlation + +> **Goal**: Fill tracing gaps in the consensus establish phase (disputes, convergence, +> threshold escalation, mode changes) and establish cross-node correlation using a +> deterministic shared trace ID derived from `previousLedger.id()`. +> +> **Approach**: Direct instrumentation in `Consensus.h` and `RCLConsensus.cpp`. +> All spans use `SpanGuard` factory methods (`span()`, `hashSpan()`, `linkedSpan()`) +> with `TraceCategory::Consensus` gating. Long-lived spans (round, establish) are +> stored as `std::optional` class members. Short-lived scoped spans +> (update_positions, check) are local variables. No macros are used — all tracing +> is via direct `SpanGuard` API calls. `SpanGuard` compiles to no-ops when +> telemetry is disabled. +> +> **Branch**: `pratik/otel-phase4-consensus-tracing` + +## Design: Switchable Correlation Strategy + +Two strategies for cross-node trace correlation, switchable via config: + +### Strategy A — Deterministic Trace ID (Default) + +Derive `trace_id = SHA256(previousLedger.id())[0:16]` so all nodes in the same +consensus round share the same trace_id without P2P context propagation. + +- **Pros**: All nodes appear in the same trace in Tempo automatically. + No collector-side post-processing needed. +- **Cons**: Overrides OTel's random trace_id generation; requires custom + `IdGenerator` or manual span context construction. + +### Strategy B — Attribute-Based Correlation + +Use normal random trace_id but attach `xrpl.consensus.ledger_id` as an attribute +on every consensus span. Correlation happens at query time via Tempo/Grafana +`by attribute` queries. + +- **Pros**: Standard OTel trace_id semantics; no SDK customization. +- **Cons**: Cross-node correlation requires query-time joins, not automatic. + +### Config + +```ini +[telemetry] +# "deterministic" (default) or "attribute" +consensus_trace_strategy=deterministic +``` + +The C++ API to query this at runtime is `Telemetry::getConsensusTraceStrategy()`, +which returns a `std::string const&` (`"deterministic"` or `"attribute"`). + +### Implementation + +In `RCLConsensus::Adaptor::startRound()`: + +- If `deterministic`: + 1. Compute `trace_id_bytes = SHA256(prevLedgerID)[0:16]` + 2. Construct `opentelemetry::trace::TraceId(trace_id_bytes)` + 3. Create a synthetic `SpanContext` with this trace_id and a random span_id: + ```cpp + auto traceId = opentelemetry::trace::TraceId(trace_id_bytes); + auto spanId = opentelemetry::trace::SpanId(random_8_bytes); + auto syntheticCtx = opentelemetry::trace::SpanContext( + traceId, spanId, opentelemetry::trace::TraceFlags(1), false); + ``` + 4. Wrap in `opentelemetry::context::Context` via + `opentelemetry::trace::SetSpan(context, syntheticSpan)` + 5. Call `startSpan("consensus.round", parentContext)` so the new span + inherits the deterministic trace_id. +- If `attribute`: start a normal `consensus.round` span, set + `xrpl.consensus.ledger_id = previousLedger.id()` as attribute. + +Both strategies always set `xrpl.consensus.round_id` (round number) and +`xrpl.consensus.ledger_id` (previous ledger hash) as attributes. + +--- + +## Design: Span Hierarchy + +``` +consensus.round (root — created in RCLConsensus::startRound, closed at accept) +│ link → previous round's SpanContext (follows-from) +│ +├── consensus.establish (phaseEstablish → acceptance, in Consensus.h) +│ ├── consensus.update_positions (each updateOurPositions call) +│ │ └── consensus.dispute.resolve (per-tx dispute resolution event) +│ ├── consensus.check (each haveConsensus call) +│ └── consensus.mode_change (short-lived span in adaptor on mode transition) +│ +├── consensus.accept (existing onAccept span — reparented under round) +│ +└── consensus.validation.send (existing — reparented, follows-from link to round) +``` + +### Span Links (follows-from relationships) + +| Link Source | Link Target | Rationale | +| ----------------------------------------- | -------------------------- | ------------------------------------------------------------------------------ | +| `consensus.round` (N+1) | `consensus.round` (N) | Causal chain: round N+1 exists because round N accepted | +| `consensus.validation.send` | `consensus.round` | Validation follows from the round that produced it; may outlive the round span | +| _(Phase 4b)_ Received proposal processing | Sender's `consensus.round` | Cross-node causal link via P2P context propagation | + +--- + +## Task 4a.0: Prerequisites — Extend SpanGuard and Telemetry APIs ✅ + +**Objective**: Add missing API surface needed by later tasks. + +**Status**: Done, but implemented differently than originally planned. The macro-based +approach (`XRPL_TRACE_CONSENSUS`, `XRPL_TRACE_ADD_EVENT`, `XRPL_TRACE_SET_ATTR`) was +**not used**. Instead, all consensus tracing uses `SpanGuard` factory methods and +direct method calls, which is cleaner and avoids macro control-flow issues. + +**What was done**: + +1. **`SpanGuard::addEvent()` with attributes** — implemented as planned: + + ```cpp + using EventAttribute = std::pair; + + void addEvent(std::string_view name, + std::initializer_list attrs); + ``` + + Callers pass plain `string_view` pairs; the implementation converts internally. + + ```cpp + // Actual usage in Consensus.h::updateOurPositions(): + span.addEvent( + "dispute.resolve", + {{consensus::span::attr::txId, to_string(txId)}, + {consensus::span::attr::disputeOurVote, dispute.getOurVote() ? "yes" : "no"}}); + ``` + +2. **Span link support** — implemented via `SpanGuard::linkedSpan()` static factory + instead of a `Telemetry::startSpan()` overload: + + ```cpp + static SpanGuard linkedSpan( + std::string_view name, SpanContext const& linkTarget); + ``` + +3. **No macros added** — `TracingInstrumentation.h` was not created. The `XRPL_TRACE_CONSENSUS`, + `XRPL_TRACE_ADD_EVENT`, and `XRPL_TRACE_SET_ATTR` macros from the original plan were + not implemented. All consensus tracing uses direct `SpanGuard` API: + - `SpanGuard::span()` — create scoped spans + - `SpanGuard::hashSpan()` — create spans with deterministic trace IDs + - `SpanGuard::linkedSpan()` — create spans with follows-from links + - `span.setAttribute()` — set attributes directly + - `span.addEvent()` — add events directly + +**Key modified files**: + +- `include/xrpl/telemetry/SpanGuard.h` — `addEvent()` overload, `EventAttribute` type alias +- `src/libxrpl/telemetry/SpanGuard.cpp` — `addEvent()` implementation + +--- + +## Task 4a.1: Adaptor `getTelemetry()` Method — NOT DONE (Not Needed) + +**Objective**: Give `Consensus.h` access to the telemetry subsystem without +coupling the generic template to OTel headers. + +**Status**: Not implemented as specified. The `getTelemetry()` adaptor method was +not needed because `SpanGuard::span()` is a static factory method that internally +checks telemetry state via the global `Telemetry` singleton. `Consensus.h` creates +spans by calling `SpanGuard::span(TraceCategory::Consensus, ...)` directly, without +needing adaptor access. Only `RCLConsensus::Adaptor` uses `app_.getTelemetry()` +directly (for `getConsensusTraceStrategy()` in `startRoundTracing()`). + +**Key insight**: The `XRPL_TRACE_*` macro approach would have required +`adaptor_.getTelemetry()`. Since macros were not used, this task became unnecessary. + +--- + +## Task 4a.2: Switchable Round Span with Deterministic Trace ID ✅ + +**Objective**: Create a `consensus.round` root span in `startRound()` that uses +the switchable correlation strategy. Store span context as a member for child +spans in `Consensus.h`. + +**Status**: Done. Implemented in `Adaptor::startRoundTracing()`. + +**What was done**: + +- `RCLConsensus::Adaptor::startRoundTracing()` helper: + - Reads `consensus_trace_strategy` via `app_.getTelemetry().getConsensusTraceStrategy()` + - **Deterministic**: uses `SpanGuard::hashSpan()` with `prevLgr.id()` data + - **Attribute**: uses `SpanGuard::span(TraceCategory::Consensus, seg::consensus, "round")` + - Sets attributes: `xrpl.consensus.ledger_id`, `xrpl.ledger.seq`, `xrpl.consensus.mode`, `trace_strategy`, `xrpl.consensus.round_id` + - Captures `roundSpanContext_` snapshot for cross-thread span linking + - Saves `prevRoundContext_` from previous round for follows-from links + +- **`SpanGuard::hashSpan()` factory**: encapsulates deterministic trace ID logic: + + ```cpp + static SpanGuard hashSpan( + TraceCategory cat, std::string_view name, + std::uint8_t const* hashData, std::size_t hashSize); + ``` + + Derives `trace_id = hashData[0:16]` so all nodes in the same round share + the same trace_id. Compiles to no-op when telemetry is disabled. + +- `consensus_trace_strategy` config parsed in `TelemetryConfig.cpp`, + stored in `Telemetry::Setup`, accessible via `Telemetry::getConsensusTraceStrategy()` + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` — `startRoundTracing()` implementation +- `src/xrpld/app/consensus/ConsensusSpanNames.h` — **(new)** compile-time span name and attribute key constants +- `include/xrpl/telemetry/Telemetry.h` — `consensusTraceStrategy` in Setup, `getConsensusTraceStrategy()` +- `src/libxrpl/telemetry/TelemetryConfig.cpp` — parse new config option + +--- + +## Task 4a.3: Span Members in `Consensus.h` ✅ + +**Objective**: Add span storage to the `Consensus` class so that spans created +in `startRound()` (adaptor) are accessible from `phaseEstablish()`, +`updateOurPositions()`, and `haveConsensus()` (template methods). + +**Status**: Done with documented plan deviation. + +**What was done**: + +- `establishSpan_` added to `Consensus` private members (as planned): + + ```cpp + std::optional establishSpan_; + ``` + +- **Plan deviation**: `roundSpan_`, `prevRoundContext_`, and `roundSpanContext_` + are stored in `RCLConsensus::Adaptor` (not `Consensus.h`) because the adaptor + has access to telemetry config for the deterministic trace ID strategy. + +- **No `#ifdef XRPL_ENABLE_TELEMETRY` guards**: Members use `std::optional` + and `SpanContext` which have no-op implementations when telemetry is disabled, + so `#ifdef` guards are unnecessary. The members are always present in the class + layout but incur negligible overhead. + +- Includes added unconditionally to `Consensus.h`: + ```cpp + #include + #include + ``` + No `TracingInstrumentation.h` include (file doesn't exist; macros not used). + +**Key modified files**: + +- `src/xrpld/consensus/Consensus.h` +- `src/xrpld/app/consensus/RCLConsensus.h` (round span and context members) + +--- + +## Task 4a.4: Instrument `phaseEstablish()` ✅ + +**Objective**: Create `consensus.establish` span wrapping the establish phase, +with attributes for convergence progress. + +**Status**: Done. Implemented via three private helpers in `Consensus.h`. + +**What was done**: + +- `startEstablishTracing()` — creates `consensus.establish` span via + `SpanGuard::span(TraceCategory::Consensus, seg::consensus, "establish")`. + Called once at start of establish phase. No `#ifdef` guards needed — + `SpanGuard::span()` returns a no-op guard when telemetry is disabled. + +- `updateEstablishTracing()` — sets attributes on each `phaseEstablish()` call: + - `converge_percent` — `convergePercent_` + - `establish_count` — `establishCounter_` + - `proposers` — `currPeerPositions_.size()` + +- `endEstablishTracing()` — calls `establishSpan_.reset()` on phase exit. + +**Key modified files**: + +- `src/xrpld/consensus/Consensus.h` — `phaseEstablish()` method + 3 helper methods + +--- + +## Task 4a.5: Instrument `updateOurPositions()` ✅ + +**Objective**: Trace each position update cycle including dispute resolution +details. + +**Status**: DONE. Span, dispute events with yays/nays, and disputes_count attribute are all implemented. + +**What was done**: + +- Creates `consensus.update_positions` scoped span via + `SpanGuard::span(TraceCategory::Consensus, seg::consensus, "update_positions")`: + + ```cpp + auto span = SpanGuard::span(TraceCategory::Consensus, seg::consensus, "update_positions"); + ``` + +- Attributes set: + - `converge_percent` — current convergence + - `proposers` — `currPeerPositions_.size()` + - `have_close_time_consensus` — close time consensus state + - `close_time_threshold` — `avCT_CONSENSUS_PCT` + - `disputes_count` — number of active disputes + +- Dispute events recorded via direct `span.addEvent()` call with yays/nays: + ```cpp + span.addEvent( + "dispute.resolve", + {{consensus::span::attr::txId, to_string(txId)}, + {consensus::span::attr::disputeOurVote, dispute.getOurVote() ? "yes" : "no"}, + {consensus::span::attr::disputeYays, std::to_string(dispute.getYays())}, + {consensus::span::attr::disputeNays, std::to_string(dispute.getNays())}}); + ``` + +**Not implemented**: + +- `proposers_agreed` / `proposers_total` attributes — not set + +**Key modified files**: + +- `src/xrpld/consensus/Consensus.h` — `updateOurPositions()` method +- `src/xrpld/consensus/DisputedTx.h` — added `getYays()` / `getNays()` (currently unused) + +--- + +## Task 4a.6: Instrument `haveConsensus()` (Threshold & Convergence) ✅ + +**Objective**: Trace consensus checking including threshold escalation. + +**Status**: DONE. The `consensus.check` span is created with all planned attributes +including the avalanche threshold. + +**What was done**: + +- Creates `consensus.check` scoped span via + `SpanGuard::span(TraceCategory::Consensus, seg::consensus, "check")`: + + ```cpp + auto span = SpanGuard::span(TraceCategory::Consensus, seg::consensus, "check"); + ``` + +- Attributes set: + - `agree_count` — peers that agree with our position + - `disagree_count` — peers that disagree + - `converge_percent` — convergence percentage + - `have_close_time_consensus` — close time consensus state + - `threshold_percent` — set to `avCT_CONSENSUS_PCT` (75%) + - `consensus_result` — "yes", "no", or "moved_on" + - `avalanche_threshold` — the escalated weight from `getNeededWeight()` on the `consensus.update_positions` span + +**Key modified files**: + +- `src/xrpld/consensus/Consensus.h` — `haveConsensus()` method + +--- + +## Task 4a.7: Instrument Mode Changes ✅ + +**Objective**: Trace consensus mode transitions (proposing ↔ observing, +wrongLedger, switchedLedger). + +**Status**: Done. + +**What was done**: + +- In `RCLConsensus::Adaptor::onModeChange()`, creates a scoped span via direct + `SpanGuard::span()` call: + + ```cpp + auto span = telemetry::SpanGuard::span( + telemetry::TraceCategory::Consensus, telemetry::seg::consensus, "mode_change"); + span.setAttribute(consensus::span::attr::modeOld, to_string(before).c_str()); // "mode_old" + span.setAttribute(consensus::span::attr::modeNew, to_string(after).c_str()); // "mode_new" + ``` + +- `MonitoredMode::set()` in `Consensus.h` calls `adaptor_.onModeChange(before, after)`. + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` — `onModeChange()` + +--- + +## Task 4a.8: Reparent Existing Spans Under Round ✅ + +**Objective**: Make existing consensus spans (`consensus.accept`, +`consensus.accept.apply`, `consensus.validation.send`) children of the +`consensus.round` root span instead of being standalone. + +**Status**: DONE. All three spans are now parented under the round span. + +**What was done**: + +- `consensus.validation.send` uses `SpanGuard::linkedSpan()` to create a + follows-from link to `roundSpanContext_`. This is thread-safe because + `roundSpanContext_` is a lightweight `SpanContext` snapshot captured on the + consensus thread and read on the jtACCEPT worker thread. + +- `consensus.accept` and `consensus.accept.apply` now use + `SpanGuard::childSpan(name, roundSpanContext_)` instead of `SpanGuard::span()` + to explicitly parent under the round span context. This solves the cross-thread + parenting problem: + - `doAccept()` runs on the jtACCEPT worker thread (not the consensus thread) + - `childSpan()` explicitly passes the parent context, bypassing OTel's + thread-local context propagation + +**Key modified files**: + +- `src/xrpld/app/consensus/RCLConsensus.cpp` + +--- + +## Task 4a.9: Build Verification and Testing ✅ + +**Objective**: Verify all Phase 4a changes compile cleanly with telemetry ON +and OFF, and don't affect consensus timing. + +**What to do**: + +1. Build with `telemetry=ON` — verify no compilation errors +2. Build with `telemetry=OFF` — verify `SpanGuard` compiles to no-ops +3. Run existing consensus unit tests +4. Verify `SpanGuard` / `SpanContext` members have negligible overhead when disabled +5. Run `pccl` pre-commit checks + +**Verification Checklist**: + +- [x] Build succeeds with telemetry ON +- [x] Build succeeds with telemetry OFF +- [x] Existing consensus tests pass +- [x] `SpanGuard` no-op path verified (no `#ifdef` needed — disabled at runtime) +- [x] No new virtual calls in hot consensus paths +- [x] `pccl` passes + +--- + +## Phase 4a Summary + +| Task | Description | Status | New Files | Modified Files | Depends On | +| ---- | ------------------------------------------------ | ------------------------ | --------- | -------------- | ---------- | +| 4a.0 | Prerequisites: extend SpanGuard & Telemetry APIs | ✅ Done (no macros) | 0 | 2 | Phase 4 | +| 4a.1 | Adaptor `getTelemetry()` method | ⏭️ Skipped (not needed) | 0 | 0 | Phase 4 | +| 4a.2 | Switchable round span with deterministic traceID | ✅ Done | 1 | 3 | 4a.0 | +| 4a.3 | Span members in `Consensus.h` | ✅ Done (with deviation) | 0 | 2 | — | +| 4a.4 | Instrument `phaseEstablish()` | ✅ Done | 0 | 1 | 4a.3 | +| 4a.5 | Instrument `updateOurPositions()` | ✅ Done | 0 | 2 | 4a.0, 4a.3 | +| 4a.6 | Instrument `haveConsensus()` (thresholds) | ✅ Done | 0 | 1 | 4a.3 | +| 4a.7 | Instrument mode changes | ✅ Done | 0 | 1 | — | +| 4a.8 | Reparent existing spans under round | ✅ Done | 0 | 1 | 4a.0, 4a.2 | +| 4a.9 | Build verification and testing | ✅ Done | 0 | 0 | 4a.0-4a.8 | + +**Parallel work**: Tasks 4a.0 and 4a.1 can run in parallel. Tasks 4a.4, 4a.5, 4a.6, and 4a.7 can run in parallel after 4a.3 (and 4a.0 for 4a.5). + +### New Spans (Phase 4a) + +| Span Name | Location | Key Attributes (actually set) | +| ---------------------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------- | +| `consensus.round` | `RCLConsensus.cpp` | `xrpl.consensus.round_id`, `xrpl.consensus.ledger_id`, `xrpl.ledger.seq`, `xrpl.consensus.mode`, `trace_strategy` | +| `consensus.establish` | `Consensus.h` | `converge_percent`, `establish_count`, `proposers` | +| `consensus.update_positions` | `Consensus.h` | `converge_percent`, `proposers`, `have_close_time_consensus`, `close_time_threshold`, `disputes_count`, `avalanche_threshold` | +| `consensus.check` | `Consensus.h` | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `consensus_result` | +| `consensus.mode_change` | `RCLConsensus.cpp` | `mode_old`, `mode_new` | + +### New Events (Phase 4a) + +| Event Name | Parent Span | Attributes (actually set) | +| ----------------- | ---------------------------- | ---------------------------------------------------------------- | +| `dispute.resolve` | `consensus.update_positions` | `xrpl.tx.id`, `dispute_our_vote`, `dispute_yays`, `dispute_nays` | +| `tx.included` | `consensus.accept.apply` | `xrpl.tx.id` | + +### New Attributes (Phase 4a) + +```cpp +// Round-level (on consensus.round) — ALL IMPLEMENTED +"xrpl.consensus.round_id" = int64 // Consensus round number +"xrpl.consensus.ledger_id" = string // previousLedger.id() hash +"trace_strategy" = string // "deterministic" or "attribute" + +// Establish-level — IMPLEMENTED +"converge_percent" = int64 // Convergence % (0-100+) +"establish_count" = int64 // Number of establish iterations +"agree_count" = int64 // Peers that agree (haveConsensus) +"disagree_count" = int64 // Peers that disagree +"threshold_percent" = int64 // Current threshold (avCT_CONSENSUS_PCT = 75%) +"consensus_result" = string // "yes", "no", "moved_on" +"have_close_time_consensus" = bool // Close time consensus reached +"close_time_threshold" = int64 // Close time voting threshold + +// Establish-level — IMPLEMENTED +"disputes_count" = int64 // Active disputes (on update_positions) +"avalanche_threshold" = int64 // Escalated weight (on update_positions) + +// Establish-level — NOT IMPLEMENTED +// "proposers_agreed" = int64 // Peers agreeing with us — not set +// "proposers_total" = int64 // Total peer positions — not set (not defined) + +// Mode change — ALL IMPLEMENTED +"mode_old" = string // Previous mode +"mode_new" = string // New mode +``` + +### Implementation Notes + +- **No macros**: The planned `XRPL_TRACE_CONSENSUS`, `XRPL_TRACE_ADD_EVENT`, and + `XRPL_TRACE_SET_ATTR` macros were not implemented. All consensus tracing uses + `SpanGuard` factory methods (`span()`, `hashSpan()`, `linkedSpan()`) and direct + method calls (`setAttribute()`, `addEvent()`). This avoids macro control-flow + issues and is cleaner than the planned approach. +- **Separation of concerns**: All non-trivial telemetry code extracted to private + helpers (`startRoundTracing`, `createValidationSpan`, `startEstablishTracing`, + `updateEstablishTracing`, `endEstablishTracing`). Business logic methods contain + single-line calls to these helpers. +- **Thread safety**: `createValidationSpan()` runs on the jtACCEPT worker thread. + Instead of accessing `roundSpan_` across threads, a `roundSpanContext_` snapshot + (lightweight `SpanContext` value type) is captured on the consensus thread in + `startRoundTracing()` and read by `createValidationSpan()`. The job queue + provides the happens-before guarantee. +- **No `#ifdef` guards**: Span members use `std::optional` and `SpanContext` + which have no-op implementations when telemetry is disabled. No `#ifdef XRPL_ENABLE_TELEMETRY` + guards needed around members or includes. +- **No `getTelemetry()` adaptor method**: `SpanGuard::span()` is a static factory that + internally checks telemetry state, so `Consensus.h` doesn't need adaptor access + for span creation. Only `RCLConsensus::Adaptor` accesses `app_.getTelemetry()` directly. +- **Config validation**: `consensus_trace_strategy` is validated to be either + `"deterministic"` or `"attribute"`, falling back to `"deterministic"` for + unrecognised values. +- **Plan deviation**: `roundSpan_` is stored in `RCLConsensus::Adaptor` (not + `Consensus.h`) because the adaptor has access to telemetry config and can + implement the deterministic trace ID strategy. `establishSpan_` is correctly + in `Consensus.h` as planned. + +--- + +# Phase 4b: Cross-Node Propagation (Future — Documentation Only) + +> **Goal**: Wire `TraceContextPropagator` for P2P messages so that proposals +> and validations carry trace context between nodes. This enables true +> distributed tracing where a proposal sent by Node A creates a child span +> on Node B. +> +> **Status**: NOT IMPLEMENTED. The protobuf fields and propagator class exist +> but are not wired. This section documents the design for future work. + +## Architecture + +``` +Node A (proposing) Node B (receiving) +───────────────── ────────────────── +consensus.round consensus.round +├── propose() ├── peerProposal() +│ └── TraceContextPropagator │ └── TraceContextPropagator +│ ::injectToProtobuf( │ ::extractFromProtobuf( +│ TMProposeSet.trace_context) │ TMProposeSet.trace_context) +│ │ └── span link → Node A's context +└── validate() └── onValidation() + └── inject into TMValidation └── extract from TMValidation +``` + +## Wiring Points + +| Message | Inject Location | Extract Location | Protobuf Field | +| --------------- | ---------------------------------- | ----------------------------------- | -------------------------- | +| `TMProposeSet` | `Adaptor::propose()` | `PeerImp::onMessage(TMProposeSet)` | field 1001: `TraceContext` | +| `TMValidation` | `Adaptor::validate()` | `PeerImp::onMessage(TMValidation)` | field 1001: `TraceContext` | +| `TMTransaction` | `NetworkOPs::processTransaction()` | `PeerImp::onMessage(TMTransaction)` | field 1001: `TraceContext` | + +## Span Link Semantics + +Received messages use **span links** (follows-from), NOT parent-child: + +- The receiver's processing span links to the sender's context +- This preserves each node's independent trace tree +- Cross-node correlation visible via linked traces in Tempo + +## Interaction with Deterministic Trace ID (Strategy A) + +When using deterministic trace_id (Phase 4a default), cross-node spans already +share the same trace_id. P2P propagation adds **span-level** linking: + +- Without propagation: spans from different nodes appear in the same trace + (same trace_id) but without parent-child or follows-from relationships. +- With propagation: spans have explicit links showing which proposal/validation + from Node A caused processing on Node B. + +## Prerequisites + +- Phase 4a (this task list) — establish phase tracing must be in place +- `TraceContextPropagator` free functions (already exist in + `include/xrpl/telemetry/TraceContextPropagator.h`) +- Protobuf `TraceContext` message (already exists, field 1001) diff --git a/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md b/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md new file mode 100644 index 00000000000..2bf33a390cb --- /dev/null +++ b/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md @@ -0,0 +1,221 @@ +# Phase 5: Integration Test Task List + +> **Goal**: End-to-end verification of the complete telemetry pipeline using a +> 6-node consensus network. Proves that RPC, transaction, and consensus spans +> flow through the observability stack (otel-collector, Tempo, Prometheus, +> Grafana) under realistic conditions. +> +> **Scope**: Integration test script, manual testing plan, 6-node local network +> setup, Tempo/Prometheus/Grafana verification. +> +> **Branch**: `pratik/otel-phase5-docs-deployment` + +### Related Plan Documents + +| Document | Relevance | +| ---------------------------------------------------------------- | ------------------------------------------ | +| [07-observability-backends.md](./07-observability-backends.md) | Tempo, Grafana, Prometheus setup | +| [05-configuration-reference.md](./05-configuration-reference.md) | Collector config, Docker Compose | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 5 tasks, definition of done | +| [Phase5_taskList.md](./Phase5_taskList.md) | Phase 5 main task list (5.6 = integration) | + +--- + +## Task IT.1: Create Integration Test Script + +**Objective**: Automated bash script that stands up a 6-node xrpld network +with telemetry, exercises all span categories, and verifies data in +Tempo/Prometheus. + +**What to do**: + +- Create `docker/telemetry/integration-test.sh`: + - Prerequisites check (docker, xrpld binary, curl, jq) + - Start observability stack via `docker compose` + - Generate 6 validator key pairs via temp standalone xrpld + - Generate 6 node configs + shared `validators.txt` + - Start 6 xrpld nodes in consensus mode (`--start`, no `-a`) + - Wait for all nodes to reach `"proposing"` state (120s timeout) + +**Key new file**: `docker/telemetry/integration-test.sh` + +**Verification**: + +- [ ] Script starts without errors +- [ ] All 6 nodes reach "proposing" state +- [ ] Observability stack is healthy (otel-collector, Tempo, Prometheus, Grafana) + +--- + +## Task IT.2: RPC Span Verification (Phase 2) + +**Objective**: Verify RPC spans flow through the telemetry pipeline. + +**What to do**: + +- Send `server_info`, `server_state`, `ledger` RPCs to node1 (port 5005) +- Wait for batch export (5s) +- Query Tempo API for: + - `rpc.request` spans (ServerHandler::onRequest) + - `rpc.process` spans (ServerHandler::processRequest) + - `rpc.command.server_info` spans (callMethod) + - `rpc.command.server_state` spans (callMethod) + - `rpc.command.ledger` spans (callMethod) +- Verify `command` attribute present on `rpc.command.*` spans + +**Verification**: + +- [ ] Tempo shows `rpc.request` traces +- [ ] Tempo shows `rpc.process` traces +- [ ] Tempo shows `rpc.command.*` traces with correct attributes + +--- + +## Task IT.3: Transaction Span Verification (Phase 3) + +**Objective**: Verify transaction spans flow through the telemetry pipeline. + +**What to do**: + +- Get genesis account sequence via `account_info` RPC +- Submit Payment transaction using genesis seed (`snoPBrXtMeMyMHUVTgbuqAfg1SUTb`) +- Wait for consensus inclusion (10s) +- Query Tempo API for: + - `tx.process` spans (NetworkOPsImp::processTransaction) on submitting node + - `tx.receive` spans (PeerImp::handleTransaction) on peer nodes +- Verify `xrpl.tx.hash` attribute on `tx.process` spans +- Verify `xrpl.peer.id` attribute on `tx.receive` spans + +**Verification**: + +- [ ] Tempo shows `tx.process` traces with `xrpl.tx.hash` +- [ ] Tempo shows `tx.receive` traces with `xrpl.peer.id` + +--- + +## Task IT.4: Consensus Span Verification (Phase 4) + +**Objective**: Verify consensus spans flow through the telemetry pipeline. + +**What to do**: + +- Consensus runs automatically in 6-node network +- Query Tempo API for: + - `consensus.proposal.send` (Adaptor::propose) + - `consensus.ledger_close` (Adaptor::onClose) + - `consensus.accept` (Adaptor::onAccept) + - `consensus.validation.send` (Adaptor::validate) +- Verify attributes: + - `xrpl.consensus.mode` on `consensus.ledger_close` + - `proposers` on `consensus.accept` + - `xrpl.consensus.ledger.seq` on `consensus.validation.send` + +**Verification**: + +- [ ] Tempo shows `consensus.ledger_close` traces with `xrpl.consensus.mode` +- [ ] Tempo shows `consensus.accept` traces with `proposers` +- [ ] Tempo shows `consensus.proposal.send` traces +- [ ] Tempo shows `consensus.validation.send` traces + +--- + +## Task IT.5: Spanmetrics Verification (Phase 5) + +**Objective**: Verify spanmetrics connector derives RED metrics from spans. + +**What to do**: + +- Query Prometheus for `traces_span_metrics_calls_total` +- Query Prometheus for `traces_span_metrics_duration_milliseconds_count` +- Verify Grafana loads at `http://localhost:3000` + +**Verification**: + +- [ ] Prometheus returns non-empty results for `traces_span_metrics_calls_total` +- [ ] Prometheus returns non-empty results for duration histogram +- [ ] Grafana UI accessible with dashboards visible + +--- + +## Task IT.6: Manual Testing Plan + +**Objective**: Document how to run tests manually for future reference. + +**What to do**: + +- Create `docker/telemetry/TESTING.md` with: + - Prerequisites section + - Single-node standalone test (quick verification) + - 6-node consensus test (full verification) + - Expected span catalog (all 11 span names with attributes) + - Verification queries (Tempo API, Prometheus API) + - Troubleshooting guide + +**Key new file**: `docker/telemetry/TESTING.md` + +**Verification**: + +- [ ] Document covers both single-node and multi-node testing +- [ ] All 11 span names documented with source file and attributes +- [ ] Troubleshooting section covers common failure modes + +--- + +## Task IT.7: Run and Verify + +**Objective**: Execute the integration test and validate results. + +**What to do**: + +- Run `docker/telemetry/integration-test.sh` locally +- Debug any failures +- Leave stack running for manual verification +- Share URLs: + - Tempo: `http://localhost:3200` + - Grafana: `http://localhost:3000` + - Prometheus: `http://localhost:9090` + +**Verification**: + +- [ ] Script completes with all checks passing +- [ ] Tempo UI shows xrpld service with all expected span names +- [ ] Grafana dashboards load and show data + +--- + +## Task IT.8: Commit + +**Objective**: Commit all new files to Phase 5 branch. + +**What to do**: + +- Run `pcc` (pre-commit checks) +- Commit 3 new files to `pratik/otel-phase5-docs-deployment` + +**Verification**: + +- [ ] `pcc` passes +- [ ] Commit created on Phase 5 branch + +--- + +## Summary + +| Task | Description | New Files | Depends On | +| ---- | ----------------------------- | --------- | ---------- | +| IT.1 | Integration test script | 1 | Phase 5 | +| IT.2 | RPC span verification | 0 | IT.1 | +| IT.3 | Transaction span verification | 0 | IT.1 | +| IT.4 | Consensus span verification | 0 | IT.1 | +| IT.5 | Spanmetrics verification | 0 | IT.1 | +| IT.6 | Manual testing plan | 1 | -- | +| IT.7 | Run and verify | 0 | IT.1-IT.6 | +| IT.8 | Commit | 0 | IT.7 | + +**Exit Criteria**: + +- [ ] All 6 xrpld nodes reach "proposing" state +- [ ] All 11 expected span names visible in Tempo +- [ ] Spanmetrics available in Prometheus +- [ ] Grafana dashboards show data +- [ ] Manual testing plan document complete diff --git a/OpenTelemetryPlan/Phase5_taskList.md b/OpenTelemetryPlan/Phase5_taskList.md new file mode 100644 index 00000000000..ed5dacc3055 --- /dev/null +++ b/OpenTelemetryPlan/Phase5_taskList.md @@ -0,0 +1,250 @@ +# Phase 5: Documentation & Deployment Task List + +> **Goal**: Production readiness — Grafana dashboards, spanmetrics pipeline, operator runbook, alert definitions, and final integration testing. This phase ensures the telemetry system is useful and maintainable in production. +> +> **Scope**: Grafana dashboard definitions, OTel Collector spanmetrics connector, Prometheus integration, alert rules, operator documentation, and production-ready Docker Compose stack. +> +> **Branch**: `pratik/otel-phase5-docs-deployment` (from `pratik/otel-phase4-consensus-tracing`) + +> **Note on attribute names**: the `xrpl..` keys shown below +> (including the collector spanmetrics dimension examples) are written in the +> older dotted form for readability — it mirrors how the fully qualified +> attribute reads in a Tempo trace view. The implemented keys follow the +> convention in [CONTRIBUTING.md](../CONTRIBUTING.md#telemetry-span-attribute-naming) +> (underscore form, e.g. `command`, `rpc_status`); the `*SpanNames.h` constants +> are the single source of truth, and the real collector dimensions must use +> those exact underscore keys (the CI naming check enforces this). + +### Related Plan Documents + +| Document | Relevance | +| ---------------------------------------------------------------- | -------------------------------------------------------------------------- | +| [07-observability-backends.md](./07-observability-backends.md) | Tempo setup (§7.1), Grafana dashboards (§7.6), alerts (§7.6.3) | +| [05-configuration-reference.md](./05-configuration-reference.md) | Collector config (§5.5), production config (§5.5.2), Docker Compose (§5.6) | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 5 tasks (§6.6), definition of done (§6.11.5) | + +--- + +## Task 5.1: Add Spanmetrics Connector to OTel Collector + +**Objective**: Derive RED metrics (Rate, Errors, Duration) from trace spans automatically, enabling Grafana time-series dashboards. + +**What to do**: + +- Edit `docker/telemetry/otel-collector-config.yaml`: + - Add `spanmetrics` connector: + ```yaml + connectors: + spanmetrics: + histogram: + explicit: + buckets: [1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 5s] + dimensions: + - name: command + - name: rpc_status + - name: consensus_phase + - name: tx_type + ``` + - Add `prometheus` exporter: + ```yaml + exporters: + prometheus: + endpoint: 0.0.0.0:8889 + ``` + - Wire the pipeline: + ```yaml + service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [debug, otlp/tempo, spanmetrics] + metrics: + receivers: [spanmetrics] + exporters: [prometheus] + ``` + +- Edit `docker/telemetry/docker-compose.yml`: + - Expose port `8889` on the collector for Prometheus scraping + - Add Prometheus service + - Add Prometheus as Grafana datasource + +**Key modified files**: + +- `docker/telemetry/otel-collector-config.yaml` +- `docker/telemetry/docker-compose.yml` + +**Key new files**: + +- `docker/telemetry/prometheus.yml` (Prometheus scrape config) +- `docker/telemetry/grafana/provisioning/datasources/prometheus.yaml` + +**Reference**: + +- [POC_taskList.md §Next Steps](./POC_taskList.md) — Metrics pipeline for Grafana dashboards + +--- + +## Task 5.2: Create Grafana Dashboards + +**Objective**: Provide pre-built Grafana dashboards for RPC performance, transaction lifecycle, and consensus health. + +**What to do**: + +- Create `docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml` (provisioning config) +- Create dashboard JSON files: + 1. **RPC Performance Dashboard** (`rpc-performance.json`): + - RPC request latency (p50/p95/p99) by command — histogram panel + - RPC throughput (requests/sec) by command — time series + - RPC error rate by command — bar gauge + - Top slowest RPC commands — table + + 2. **Transaction Overview Dashboard** (`transaction-overview.json`): + - Transaction processing rate — time series + - Transaction latency distribution — histogram + - Suppression rate (duplicates) — stat panel + - Transaction processing path (sync vs async) — pie chart + + 3. **Consensus Health Dashboard** (`consensus-health.json`): + - Consensus round duration — time series + - Phase duration breakdown (open/establish/accept) — stacked bar + - Proposals sent/received per round — stat panel + - Consensus mode distribution (proposing/observing) — pie chart + +- Store dashboards in `docker/telemetry/grafana/dashboards/` + +**Key new files**: + +- `docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml` +- `docker/telemetry/grafana/dashboards/rpc-performance.json` +- `docker/telemetry/grafana/dashboards/transaction-overview.json` +- `docker/telemetry/grafana/dashboards/consensus-health.json` + +**Reference**: + +- [07-observability-backends.md §7.6](./07-observability-backends.md) — Grafana dashboard specifications +- [01-architecture-analysis.md §1.8.3](./01-architecture-analysis.md) — Dashboard panel examples + +--- + +## Task 5.3: Define Alert Rules + +**Objective**: Create alert definitions for key telemetry anomalies. + +**What to do**: + +- Create `docker/telemetry/grafana/provisioning/alerting/alerts.yaml`: + - **RPC Latency Alert**: p99 latency > 1s for any command over 5 minutes + - **RPC Error Rate Alert**: Error rate > 5% for any command over 5 minutes + - **Consensus Duration Alert**: Round duration > 10s (warn), > 30s (critical) + - **Transaction Processing Alert**: Processing rate drops below threshold + - **Telemetry Pipeline Health**: No spans received for > 2 minutes + +**Key new files**: + +- `docker/telemetry/grafana/provisioning/alerting/alerts.yaml` + +**Reference**: + +- [07-observability-backends.md §7.6.3](./07-observability-backends.md) — Alert rule definitions + +--- + +## Task 5.4: Production Collector Configuration + +**Objective**: Create a production-ready OTel Collector configuration with tail-based sampling and resource limits. + +**What to do**: + +- Create `docker/telemetry/otel-collector-config-production.yaml`: + - Tail-based sampling policy: + - Always sample errors and slow traces + - 10% base sampling rate for normal traces + - Always sample first trace for each unique RPC command + - Resource limits: + - Memory limiter processor (80% of available memory) + - Queued retry for export failures + - TLS configuration for production endpoints + - Health check endpoint + +**Key new files**: + +- `docker/telemetry/otel-collector-config-production.yaml` + +**Reference**: + +- [05-configuration-reference.md §5.5.2](./05-configuration-reference.md) — Production collector config + +--- + +## Task 5.5: Operator Runbook + +**Objective**: Create operator documentation for managing the telemetry system in production. + +**What to do**: + +- Create `docs/telemetry-runbook.md`: + - **Setup**: How to enable telemetry in xrpld + - **Configuration**: All config options with descriptions + - **Collector Deployment**: Docker Compose vs. Kubernetes vs. bare metal + - **Troubleshooting**: Common issues and resolutions + - No traces appearing + - High memory usage from telemetry + - Collector connection failures + - Sampling configuration tuning + - **Performance Tuning**: Batch size, queue size, sampling ratio guidelines + - **Upgrading**: How to upgrade OTel SDK and Collector versions + +**Key new files**: + +- `docs/telemetry-runbook.md` + +--- + +## Task 5.6: Final Integration Testing + +**Objective**: Validate the complete telemetry stack end-to-end. + +**What to do**: + +1. Start full Docker stack (Collector, Tempo, Grafana, Prometheus) +2. Build xrpld with `telemetry=ON` +3. Run in standalone mode with telemetry enabled +4. Generate RPC traffic and verify traces in Tempo +5. Verify dashboards populate in Grafana +6. Verify alerts trigger correctly +7. Test telemetry OFF path (no regressions) +8. Run full test suite + +**Verification Checklist**: + +- [ ] Docker stack starts without errors +- [ ] Traces appear in Tempo with correct hierarchy +- [ ] Grafana dashboards show metrics derived from spans +- [ ] Prometheus scrapes spanmetrics successfully +- [ ] Alerts can be triggered by simulated conditions +- [ ] Build succeeds with telemetry ON and OFF +- [ ] Full test suite passes + +--- + +## Summary + +| Task | Description | New Files | Modified Files | Depends On | +| ---- | ---------------------------------- | --------- | -------------- | ---------- | +| 5.1 | Spanmetrics connector + Prometheus | 2 | 2 | Phase 4 | +| 5.2 | Grafana dashboards | 4 | 0 | 5.1 | +| 5.3 | Alert definitions | 1 | 0 | 5.1 | +| 5.4 | Production collector config | 1 | 0 | Phase 4 | +| 5.5 | Operator runbook | 1 | 0 | Phase 4 | +| 5.6 | Final integration testing | 0 | 0 | 5.1-5.5 | + +**Parallel work**: Tasks 5.1, 5.4, and 5.5 can run in parallel. Tasks 5.2 and 5.3 depend on 5.1. Task 5.6 depends on all others. + +**Exit Criteria** (from [06-implementation-phases.md §6.11.5](./06-implementation-phases.md)): + +- [ ] Dashboards deployed and showing data +- [ ] Alerts configured and tested +- [ ] Operator documentation complete +- [ ] Production collector config ready +- [ ] Full test suite passes diff --git a/OpenTelemetryPlan/Phase7_taskList.md b/OpenTelemetryPlan/Phase7_taskList.md new file mode 100644 index 00000000000..971372fdb0e --- /dev/null +++ b/OpenTelemetryPlan/Phase7_taskList.md @@ -0,0 +1,587 @@ +# Phase 7: Native OTel Metrics Migration — Task List + +> **Goal**: Replace `StatsDCollector` with a native OpenTelemetry Metrics SDK implementation behind the existing `beast::insight::Collector` interface, eliminating the StatsD UDP dependency. +> +> **Scope**: New `OTelCollectorImpl` class, `CollectorManager` config change, OTel Collector pipeline update, Grafana dashboard metric name migration, integration tests. +> +> **Branch**: `pratik/otel-phase7-native-metrics` (from `pratik/otel-phase6-statsd`) + +### Related Plan Documents + +| Document | Relevance | +| -------------------------------------------------------------------- | --------------------------------------------------------------- | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 7 plan: motivation, architecture, exit criteria (§6.8) | +| [02-design-decisions.md](./02-design-decisions.md) | Collector interface design, beast::insight coexistence strategy | +| [05-configuration-reference.md](./05-configuration-reference.md) | `[insight]` and `[telemetry]` config sections | +| [09-data-collection-reference.md](./09-data-collection-reference.md) | Complete metric inventory that must be preserved | + +--- + +## Task 7.1: Add OTel Metrics SDK to Build Dependencies + +**Objective**: Enable the OTel C++ Metrics SDK components in the build system. + +**What to do**: + +- Edit `conanfile.py`: + - Add OTel metrics SDK components to the dependency list when `telemetry=True` + - Components needed: `opentelemetry-cpp::metrics`, `opentelemetry-cpp::otlp_http_metric_exporter` + +- Edit `CMakeLists.txt` (telemetry section): + - Link `opentelemetry::metrics` and `opentelemetry::otlp_http_metric_exporter` targets + +**Key modified files**: + +- `conanfile.py` +- `CMakeLists.txt` (or the relevant telemetry cmake target) + +**Reference**: [05-configuration-reference.md §5.3](./05-configuration-reference.md) — CMake integration + +--- + +## Task 7.2: Implement OTelCollector Class + +**Objective**: Create the core `OTelCollector` implementation that maps beast::insight instruments to OTel Metrics SDK instruments. + +**What to do**: + +- Create `include/xrpl/beast/insight/OTelCollector.h`: + - Public factory: `static std::shared_ptr New(std::string const& endpoint, std::string const& prefix, beast::Journal journal)` + - Derives from `StatsDCollector` (or directly from `Collector` — TBD based on shared code) + +- Create `src/libxrpl/beast/insight/OTelCollector.cpp` (~400-500 lines): + - **OTelCounterImpl**: Wraps `opentelemetry::metrics::Counter`. `increment(amount)` calls `counter->Add(amount)`. + - **OTelGaugeImpl**: Uses `opentelemetry::metrics::ObservableGauge` with an async callback. `set(value)` stores value atomically; callback reads it during collection. + - **OTelMeterImpl**: Wraps `opentelemetry::metrics::Counter`. `increment(amount)` calls `counter->Add(amount)`. Semantically identical to Counter but unsigned. + - **OTelEventImpl**: Wraps `opentelemetry::metrics::Histogram`. `notify(duration)` calls `histogram->Record(duration.count())`. Uses explicit bucket boundaries matching SpanMetrics: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms. + - **OTelHookImpl**: Stores handler function. Called during periodic metric collection (same 1s pattern via PeriodicMetricReader). + - **OTelCollectorImp**: Main class. + - Creates `MeterProvider` with `PeriodicMetricReader` (1s export interval) + - Creates `OtlpHttpMetricExporter` pointing to `[telemetry]` endpoint + - Sets resource attributes (service.name, service.instance.id) matching trace exporter + - Implements all `make_*()` factory methods + - Prefixes metric names with `[insight] prefix=` value + +- Guard all OTel SDK includes with `#ifdef XRPL_ENABLE_TELEMETRY` to compile to `NullCollector` equivalents when telemetry disabled. + +**Key new files**: + +- `include/xrpl/beast/insight/OTelCollector.h` +- `src/libxrpl/beast/insight/OTelCollector.cpp` + +**Key patterns to follow**: + +- Match `StatsDCollector.cpp` structure: private impl classes, intrusive list for metrics, strand-based thread safety +- Match existing telemetry code style from `src/libxrpl/telemetry/Telemetry.cpp` +- Use RAII for MeterProvider lifecycle (shutdown on destructor) + +**Reference**: [04-code-samples.md](./04-code-samples.md) — code style and patterns + +--- + +## Task 7.3: Update CollectorManager + +**Objective**: Add `server=otel` config option to route metric creation to the new OTel backend. + +**What to do**: + +- Edit `src/xrpld/app/main/CollectorManager.cpp`: + - In the constructor, add a third branch after `server == "statsd"`: + ```cpp + else if (server == "otel") + { + // Read endpoint from [telemetry] section + auto const endpoint = get(telemetryParams, "endpoint", + "http://localhost:4318/v1/metrics"); + std::string const& prefix(get(params, "prefix")); + collector_ = beast::insight::OTelCollector::New( + endpoint, prefix, journal); + } + ``` + - This requires access to the `[telemetry]` config section — may need to pass it as a parameter or read from Application config. + +- Edit `src/xrpld/app/main/CollectorManager.h`: + - Add `#include ` + +**Key modified files**: + +- `src/xrpld/app/main/CollectorManager.cpp` +- `src/xrpld/app/main/CollectorManager.h` + +--- + +## Task 7.4: Update OTel Collector Configuration + +**Objective**: Add a metrics pipeline to the OTLP receiver and remove the StatsD receiver dependency. + +**What to do**: + +- Edit `docker/telemetry/otel-collector-config.yaml`: + - Remove `statsd` receiver (no longer needed when `server=otel`) + - Add metrics pipeline under `service.pipelines`: + ```yaml + metrics: + receivers: [otlp, spanmetrics] + processors: [batch] + exporters: [prometheus] + ``` + - The OTLP receiver already listens on :4318 — it just needs to be added to the metrics pipeline receivers. + - Keep `spanmetrics` connector in the metrics pipeline so span-derived RED metrics continue working. + +- Edit `docker/telemetry/docker-compose.yml`: + - Remove UDP :8125 port mapping from otel-collector service + - Update xrpld service config: change `[insight] server=statsd` to `server=otel` + +**Key modified files**: + +- `docker/telemetry/otel-collector-config.yaml` +- `docker/telemetry/docker-compose.yml` + +**Note**: Keep a commented-out `statsd` receiver block for operators who need backward compatibility. + +--- + +## Task 7.5: Preserve Metric Names in Prometheus + +**Objective**: Ensure existing Grafana dashboards continue working with identical metric names. + +**What to do**: + +- In `OTelCollector.cpp`, construct OTel instrument names to match existing Prometheus metric names: + - beast::insight `make_gauge("LedgerMaster", "Validated_Ledger_Age")` → OTel instrument name: `xrpld_LedgerMaster_Validated_Ledger_Age` + - The prefix + group + name concatenation must produce the same string as `StatsDCollector`'s format + - Use underscores as separators (matching StatsD convention) + +- Verify in integration test that key Prometheus queries still return data: + - `xrpld_LedgerMaster_Validated_Ledger_Age` + - `xrpld_Peer_Finder_Active_Inbound_Peers` + - `xrpld_rpc_requests` + +**Key consideration**: OTel Prometheus exporter may normalize metric names differently than StatsD receiver. Test this early (Task 7.2) and adjust naming strategy if needed. The OTel SDK's Prometheus exporter adds `_total` suffix to counters and converts dots to underscores — match existing conventions. + +--- + +## Task 7.6: Update Grafana Dashboards + +**Objective**: Update the 3 StatsD dashboards if any metric names change due to OTLP export format differences. + +**What to do**: + +- If Task 7.5 confirms metric names are preserved exactly, no dashboard changes needed. +- If OTLP export produces different names (e.g., `_total` suffix on counters), update: + - `docker/telemetry/grafana/dashboards/statsd-node-health.json` + - `docker/telemetry/grafana/dashboards/statsd-network-traffic.json` + - `docker/telemetry/grafana/dashboards/statsd-rpc-pathfinding.json` +- Rename dashboard titles from "StatsD" to "System Metrics" or similar (since they're no longer StatsD-sourced). + +**Key modified files**: + +- `docker/telemetry/grafana/dashboards/statsd-*.json` (3 files, conditionally) + +--- + +## Task 7.7: Update Integration Tests + +**Objective**: Verify the full OTLP metrics pipeline end-to-end. + +**What to do**: + +- Edit `docker/telemetry/integration-test.sh`: + - Update test config to use `[insight] server=otel` + - Verify metrics arrive in Prometheus via OTLP (not StatsD) + - Add check that StatsD receiver is no longer required + - Preserve all existing metric presence checks + +**Key modified files**: + +- `docker/telemetry/integration-test.sh` + +--- + +## Task 7.8: Update Documentation + +**Objective**: Update all plan docs, runbook, and reference docs to reflect the migration. + +**What to do**: + +- Edit `docs/telemetry-runbook.md`: + - Update `[insight]` config examples to show `server=otel` + - Update troubleshooting section (no more StatsD UDP debugging) + +- Edit `OpenTelemetryPlan/09-data-collection-reference.md`: + - Update Data Flow Overview diagram (remove StatsD receiver) + - Update Section 2 header from "StatsD Metrics" to "System Metrics (OTel native)" + - Update config examples + +- Edit `OpenTelemetryPlan/05-configuration-reference.md`: + - Add `server=otel` option to `[insight]` section docs + +- Edit `docker/telemetry/TESTING.md`: + - Update setup instructions to use `server=otel` + +**Key modified files**: + +- `docs/telemetry-runbook.md` +- `OpenTelemetryPlan/09-data-collection-reference.md` +- `OpenTelemetryPlan/05-configuration-reference.md` +- `docker/telemetry/TESTING.md` + +--- + +## Task 7.9: ValidationTracker — Validation Agreement Computation + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — the most valuable metric from the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). +> +> **Upstream**: Phase 4 Task 4.8 (validation span attributes provide ledger hash context). +> **Downstream**: Phase 9 (Validator Health dashboard), Phase 10 (validation checks), Phase 11 (agreement alert rules). + +**Objective**: Implement a stateful class that tracks whether our validator's validations agree with network consensus, maintaining rolling 1h and 24h windows with an 8-second grace period and 5-minute late repair window. + +**Architecture**: + +``` +consensus.validation.send ────> ValidationTracker ────> MetricsRegistry +(records our validation (reconciles after (exports agreement + for ledger X) 8s grace period) gauges every 10s) + +ledger.validate ──────────────> ValidationTracker +(records which ledger (marks ledger X as + network validated) agreed or missed) +``` + +**What to do**: + +- Create `src/xrpld/telemetry/ValidationTracker.h`: + - `recordOurValidation(ledgerHash, ledgerSeq)` — called when we send a validation + - `recordNetworkValidation(ledgerHash, seq)` — called when a ledger is fully validated + - `reconcile()` — called periodically; reconciles pending ledger events after 8s grace period + - Getters: `agreementPct1h()`, `agreementPct24h()`, `agreements1h()`, `missed1h()`, `agreements24h()`, `missed24h()`, `totalAgreements()`, `totalMissed()`, `totalValidationsSent()`, `totalValidationsChecked()` + - Thread-safety: atomics for counters, mutex for window deques + +- Create `src/xrpld/telemetry/detail/ValidationTracker.cpp`: + - Reconciliation logic: after 8s grace period, check if `weValidated && networkValidated && sameHash` → agreement; else missed + - Late repair: if a late validation arrives within 5 minutes, correct a false-positive miss + - Sliding window: `std::deque` evicts entries older than 1h/24h on each reconciliation pass + - Ring buffer of 1000 `LedgerEvent` structs for pending reconciliation + +- Add recording hooks (modifying Phase 4 code from Phase 7 branch): + - `RCLConsensus.cpp` `validate()`: call `tracker.recordOurValidation()` + - `LedgerMaster.cpp` fully-validated path: call `tracker.recordNetworkValidation()` + +**Key data structures**: + +```cpp +struct LedgerEvent { + uint256 ledgerHash; + LedgerIndex seq; + TimePoint closeTime; + bool weValidated = false; + bool networkValidated = false; + bool reconciled = false; + bool agreed = false; +}; + +struct WindowEvent { + TimePoint time; + bool agreed; +}; +``` + +**Key new files**: + +- `src/xrpld/telemetry/ValidationTracker.h` +- `src/xrpld/telemetry/detail/ValidationTracker.cpp` + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.h` (add ValidationTracker member) +- `src/xrpld/telemetry/MetricsRegistry.cpp` (add gauge callback reading from tracker) +- `src/xrpld/app/consensus/RCLConsensus.cpp` (add recording hooks) +- `src/xrpld/app/ledger/detail/LedgerMaster.cpp` (add recording hook) + +**Exit Criteria**: + +- [ ] ValidationTracker correctly tracks agreement with 8s grace period +- [ ] 5-minute late repair corrects false-positive misses +- [ ] Thread-safe (atomics + mutex for window deques) +- [ ] Rolling windows correctly evict stale entries +- [ ] Unit tests: normal agreement, missed validation, late repair, window eviction + +--- + +## Task 7.10: Validator Health Observable Gauges + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Export amendment blocked, UNL health, and quorum data as a native OTel observable gauge. + +**What to do**: + +- In `MetricsRegistry.cpp` `registerAsyncGauges()`, add: + +```cpp +validatorHealthGauge_ = meter_->CreateDoubleObservableGauge( + "xrpld_validator_health", "Validator health indicators"); +``` + +**Gauge label values**: + +| Label `metric=` | Type | Source | +| ------------------- | ------ | ------------------------------------------------- | +| `amendment_blocked` | int64 | `app_.getOPs().isAmendmentBlocked()` → 0/1 | +| `unl_blocked` | int64 | `app_.getOPs().isUNLBlocked()` → 0/1 | +| `unl_expiry_days` | double | `app_.validators().expires()` → days until expiry | +| `validation_quorum` | int64 | `app_.validators().quorum()` | + +### Sub-task 7.10a: Per-Validator Validation Count (Flag Ledger Window) + +**Objective**: Track how many ledgers each UNL validator has validated over +the last 256 consecutive ledgers (one flag ledger window). This is the key +UNL participation metric — validators consistently below threshold may be +candidates for removal from the UNL. + +**What to do**: + +- Add a new observable gauge: + +```cpp +validatorParticipationGauge_ = meter_->CreateInt64ObservableGauge( + "xrpld_validator_participation", + "Per-validator validation count over the last 256 ledgers"); +``` + +- The callback queries `app_.getValidations()` to get the trusted + validation set for each of the last 256 ledger hashes (from + `LedgerMaster::getValidatedLedger()` walking backwards). For each + validator public key in the UNL, count how many of those 256 ledgers + have a matching validation. + +- **Label dimensions**: + - `validator` — base58-encoded validator master public key + - `service_instance_id` — this node's identity (standard) + +- **Emission**: every flag ledger (256 ledgers, ~15 minutes) or on a + 10-second async gauge callback with cached results (recompute only + at flag ledger boundaries). + +- **Data source**: `RCLValidations::getTrustedForLedger(hash, seq)` returns + `std::vector>` with `getSignerPublic()` + for each. The UNL list is from `app_.getValidators().getTrustedMasterKeys()`. + +- **Dashboard panel**: Add a table panel to the Validator Health dashboard + showing `xrpld_validator_participation` grouped by `validator` label, + with a threshold color (green >= 240, yellow >= 200, red < 200). + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` + +**Exit Criteria**: + +- [ ] Gauge emits one time series per UNL validator +- [ ] Values range 0-256 and update at flag ledger boundaries +- [ ] Grafana table panel shows per-validator participation +- [ ] Validators below 75% participation are highlighted in red + +--- + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` + +**Exit Criteria**: + +- [ ] All 4 base label values emitted every 10s +- [ ] `unl_expiry_days` is negative when expired, positive when active +- [ ] Per-validator participation gauge emits at flag ledger boundaries +- [ ] Values visible in Prometheus + +--- + +## Task 7.11: Peer Quality Observable Gauges + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Export peer health aggregates (latency P90, insane peers, version awareness) as a native OTel observable gauge. + +**What to do**: + +- In `MetricsRegistry.cpp` `registerAsyncGauges()`, add a callback that iterates `app_.overlay().foreach(...)` to: + - Collect per-peer latency values, sort, compute P90 + - Count peers with `tracking_ == diverged` (insane) + - Compare peer `getVersion()` to own version for upgrade awareness + +**Gauge label values**: + +| Label `metric=` | Type | Source | +| -------------------------- | ------ | ------------------------------------- | +| `peer_latency_p90_ms` | double | P90 from sorted peer latencies | +| `peers_insane_count` | int64 | Peers with diverged tracking status | +| `peers_higher_version_pct` | double | % of peers on newer xrpld version | +| `upgrade_recommended` | int64 | 1 if `peers_higher_version_pct > 60%` | + +**Implementation note**: The callback runs every 10s on the metrics reader thread. Iterating ~50-200 peers is acceptable overhead. + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` + +**Exit Criteria**: + +- [ ] P90 latency computed correctly +- [ ] Insane count matches `peers` RPC output +- [ ] Version comparison handles format variations (e.g., "xrpld-2.4.0-rc1") + +--- + +## Task 7.12: Ledger Economy Observable Gauges + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Export fee, reserve, ledger age, and transaction rate as a native OTel observable gauge. + +**Gauge label values**: + +| Label `metric=` | Type | Source | +| -------------------- | ------ | --------------------------------------------------- | +| `base_fee_xrp` | double | Base fee from validated ledger fee settings (drops) | +| `reserve_base_xrp` | double | Account reserve from validated ledger (drops) | +| `reserve_inc_xrp` | double | Owner reserve increment (drops) | +| `ledger_age_seconds` | double | `now - lastValidatedCloseTime` | +| `transaction_rate` | double | Derived: tx count delta / time delta (smoothed) | + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` + +**Exit Criteria**: + +- [ ] Fee values match `server_info` RPC output +- [ ] `ledger_age_seconds` increases monotonically between ledger closes +- [ ] `transaction_rate` is smoothed (rolling average) + +--- + +## Task 7.13: State Tracking Observable Gauges + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Export extended state value (0-6 encoding combining OperatingMode + ConsensusMode) and time-in-current-state. + +**Gauge label values**: + +| Label `metric=` | Type | Source | +| ------------------------------- | ------ | ----------------------------------------------- | +| `state_value` | int64 | 0-6 encoding (see spec for mapping) | +| `time_in_current_state_seconds` | double | `now - lastModeChangeTime` from StateAccounting | + +**State value encoding**: 0=disconnected, 1=connected, 2=syncing, 3=tracking, 4=full, 5=validating (full + validating), 6=proposing (full + proposing). + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` + +**Exit Criteria**: + +- [ ] `state_value` correctly combines OperatingMode and ConsensusMode +- [ ] `time_in_current_state_seconds` resets on mode change + +--- + +## Task 7.14: Storage Detail and Sync Info Gauges + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Export NuDB-specific storage size and initial sync duration. + +**Gauge label values**: + +| Gauge Name | Label `metric=` | Type | Source | +| ---------------------- | ------------------------------- | ------ | ----------------------------- | +| `xrpld_storage_detail` | `nudb_bytes` | int64 | NuDB backend file size | +| `xrpld_sync_info` | `initial_sync_duration_seconds` | double | Time from start to first FULL | + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` + +**Exit Criteria**: + +- [ ] NuDB file size reported in bytes (0 if NuDB not configured) +- [ ] Sync duration captured once and remains stable after reaching FULL + +--- + +## Task 7.15: New Synchronous Counters + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Add 7 new event counters incremented at their respective instrumentation sites. + +| Counter Name | Increment Site | Source File | +| ----------------------------------- | -------------------------------- | --------------------- | +| `xrpld_ledgers_closed_total` | `onAccept()` in consensus | RCLConsensus.cpp | +| `xrpld_validations_sent_total` | `validate()` in consensus | RCLConsensus.cpp | +| `xrpld_validations_checked_total` | Network validation received | LedgerMaster.cpp | +| `xrpld_validation_agreements_total` | ValidationTracker reconciliation | ValidationTracker.cpp | +| `xrpld_validation_missed_total` | ValidationTracker reconciliation | ValidationTracker.cpp | +| `xrpld_state_changes_total` | `setMode()` in NetworkOPs | NetworkOPs.cpp | +| `xrpld_jq_trans_overflow_total` | Job queue overflow path | JobQueue.cpp | + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (declarations), plus recording sites in RCLConsensus.cpp, LedgerMaster.cpp, NetworkOPs.cpp, JobQueue.cpp + +**Exit Criteria**: + +- [ ] All 7 counters monotonically increase during normal operation +- [ ] Counter values match expected rates (e.g., ledgers_closed ≈ 1 per 3-5s) + +--- + +## Task 7.16: Validation Agreement Observable Gauge + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Export rolling window agreement stats from `ValidationTracker` (Task 7.9). + +**Gauge label values**: + +| Gauge Name | Label `metric=` | Type | Source | +| ---------------------------- | ------------------- | ------ | --------------------------- | +| `xrpld_validation_agreement` | `agreement_pct_1h` | double | `tracker.agreementPct1h()` | +| | `agreements_1h` | int64 | `tracker.agreements1h()` | +| | `missed_1h` | int64 | `tracker.missed1h()` | +| | `agreement_pct_24h` | double | `tracker.agreementPct24h()` | +| | `agreements_24h` | int64 | `tracker.agreements24h()` | +| | `missed_24h` | int64 | `tracker.missed24h()` | + +**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Exit Criteria**: + +- [ ] Agreement percentages in range [0.0, 100.0] +- [ ] Window stats stabilize after 1h/24h of operation + +--- + +## Summary Table + +| Task | Description | New Files | Modified Files | Depends On | +| ---- | -------------------------------------- | --------- | -------------- | ---------- | +| 7.1 | Add OTel Metrics SDK to build deps | 0 | 2 | — | +| 7.2 | Implement OTelCollector class | 2 | 0 | 7.1 | +| 7.3 | Update CollectorManager config routing | 0 | 2 | 7.2 | +| 7.4 | Update OTel Collector YAML and Docker | 0 | 2 | 7.3 | +| 7.5 | Preserve metric names in Prometheus | 0 | 1 | 7.2 | +| 7.6 | Update Grafana dashboards (if needed) | 0 | 3 | 7.5 | +| 7.7 | Update integration tests | 0 | 1 | 7.4 | +| 7.8 | Update documentation | 0 | 4 | 7.6 | +| 7.9 | ValidationTracker (agreement tracking) | 2 | 4 | 7.2, P4.8 | +| 7.10 | Validator health observable gauges | 0 | 2 | 7.2 | +| 7.11 | Peer quality observable gauges | 0 | 2 | 7.2 | +| 7.12 | Ledger economy observable gauges | 0 | 2 | 7.2 | +| 7.13 | State tracking observable gauges | 0 | 2 | 7.2 | +| 7.14 | Storage detail and sync info gauges | 0 | 2 | 7.2 | +| 7.15 | New synchronous counters | 0 | 6 | 7.2 | +| 7.16 | Validation agreement observable gauge | 0 | 1 | 7.9 | + +**Parallel work**: Tasks 7.4 and 7.5 can run in parallel after 7.2/7.3 complete. Task 7.6 depends on 7.5's findings. Tasks 7.7 and 7.8 can run in parallel after 7.6. Tasks 7.10-7.14 can all run in parallel after 7.2. Task 7.15 depends on 7.2. Task 7.16 depends on 7.9. Task 7.9 depends on 7.2 and Phase 4 Task 4.8. + +**Exit Criteria** (from [06-implementation-phases.md §6.8](./06-implementation-phases.md)): + +- [ ] All 255+ metrics visible in Prometheus via OTLP pipeline (no StatsD receiver) +- [ ] `server=otel` is the default in development docker-compose +- [ ] `server=statsd` still works as a fallback +- [ ] Existing Grafana dashboards display data correctly +- [ ] Integration test passes with OTLP-only metrics pipeline +- [ ] No performance regression vs StatsD baseline (< 1% CPU overhead) +- [ ] Deferred Task 6.1 (`|m` wire format) no longer relevant — Meter mapped to OTel Counter +- [ ] ValidationTracker agreement % stabilizes after 1h under normal consensus +- [ ] All new gauges and counters visible in Prometheus with non-zero values diff --git a/OpenTelemetryPlan/Phase8_taskList.md b/OpenTelemetryPlan/Phase8_taskList.md new file mode 100644 index 00000000000..78499908582 --- /dev/null +++ b/OpenTelemetryPlan/Phase8_taskList.md @@ -0,0 +1,239 @@ +# Phase 8: Log-Trace Correlation and Centralized Log Ingestion — Task List + +> **Goal**: Inject trace context (trace_id, span_id) into xrpld's Journal log output for log-trace correlation, and add OTel Collector filelog receiver to ingest logs into Grafana Loki for unified observability. +> +> **Scope**: Two independent sub-phases — 8a (code change: trace_id in logs) and 8b (infra only: filelog receiver to Loki). No changes to the `beast::Journal` public API. +> +> **Branch**: `pratik/otel-phase8-log-correlation` (from `pratik/otel-phase7-native-metrics`) + +### Related Plan Documents + +| Document | Relevance | +| ---------------------------------------------------------------- | -------------------------------------------------------------- | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 8 plan: motivation, architecture, exit criteria (§6.8.1) | +| [07-observability-backends.md](./07-observability-backends.md) | Loki backend recommendation, Grafana data source provisioning | +| [Phase7_taskList.md](./Phase7_taskList.md) | Prerequisite — native OTel metrics pipeline must be working | +| [05-configuration-reference.md](./05-configuration-reference.md) | `[telemetry]` config (trace_id injection toggle) | + +--- + +## Task 8.1: Inject trace_id into Logs::format() + +**Objective**: Add OTel trace context to every log line that is emitted within an active span. + +**What to do**: + +- Edit `src/libxrpl/basics/Log.cpp`: + - In `Logs::format()` (around line 346), after severity is appended, check for active OTel span. The implementation checks the context value directly to avoid the heap allocation that `GetSpan()` performs on the no-span path: + ```cpp + #ifdef XRPL_ENABLE_TELEMETRY + { + auto context = opentelemetry::context::RuntimeContext::GetCurrent(); + auto spanValue = context.GetValue(opentelemetry::trace::kSpanKey); + if (opentelemetry::nostd::holds_alternative< + opentelemetry::nostd::shared_ptr>(spanValue)) + { + auto span = opentelemetry::nostd::get< + opentelemetry::nostd::shared_ptr>(spanValue); + auto spanCtx = span->GetContext(); + if (spanCtx.IsValid()) + { + char traceId[32], spanId[16]; + spanCtx.trace_id().ToLowerBase16( + opentelemetry::nostd::span{traceId}); + spanCtx.span_id().ToLowerBase16( + opentelemetry::nostd::span{spanId}); + output += "trace_id="; + output.append(traceId, 32); + output += " span_id="; + output.append(spanId, 16); + output += ' '; + } + } + } + #endif + ``` + - Add `#include` for OTel context headers, guarded by `#ifdef XRPL_ENABLE_TELEMETRY` + +- Edit `include/xrpl/basics/Log.h`: + - No changes needed — format() signature unchanged + +**Key modified files**: + +- `src/libxrpl/basics/Log.cpp` + +**Performance note**: The implementation checks the thread-local context value directly (avoiding the heap allocation that `GetSpan()` performs on the no-span path). On threads without an active span (~99% of log lines), the cost is a thread-local read + variant type check (~15-20ns). On the active-span path, an additional shared_ptr copy + `GetContext()` + `IsValid()` adds ~50ns total. Overhead is negligible at typical logging rates. + +--- + +## Task 8.2: Add Loki to Docker Compose Stack + +**Objective**: Add Grafana Loki as a log storage backend in the development observability stack. + +**What to do**: + +- Edit `docker/telemetry/docker-compose.yml`: + - Add Loki service: + ```yaml + loki: + image: grafana/loki:3.4.2 + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + ``` + - Add Loki as a Grafana data source in provisioning + +- Create `docker/telemetry/grafana/provisioning/datasources/loki.yaml`: + - Configure Loki data source with derived fields linking `trace_id` to Tempo + +**Key new files**: + +- `docker/telemetry/grafana/provisioning/datasources/loki.yaml` + +**Key modified files**: + +- `docker/telemetry/docker-compose.yml` + +--- + +## Task 8.3: Add Filelog Receiver to OTel Collector + +**Objective**: Configure the OTel Collector to tail xrpld's log file and export to Loki. + +**What to do**: + +- Edit `docker/telemetry/otel-collector-config.yaml`: + - Add `filelog` receiver: + ```yaml + receivers: + filelog: + include: [/var/log/xrpld/*/debug.log] + operators: + - type: regex_parser + regex: '^(?P\S+)\s+(?P\S+):(?P\S+)\s+(?:trace_id=(?P[a-f0-9]+)\s+span_id=(?P[a-f0-9]+)\s+)?(?P.*)$' + timestamp: + parse_from: attributes.timestamp + layout: "%Y-%m-%dT%H:%M:%S.%fZ" + ``` + - Add logs pipeline: + ```yaml + service: + pipelines: + logs: + receivers: [filelog] + processors: [batch] + exporters: [otlp/loki] + ``` + - Add Loki exporter: + ```yaml + exporters: + otlphttp/loki: + endpoint: http://loki:3100/otlp + ``` + +- Mount xrpld's log directory into the collector container via docker-compose volume + +**Key modified files**: + +- `docker/telemetry/otel-collector-config.yaml` +- `docker/telemetry/docker-compose.yml` + +--- + +## Task 8.4: Configure Grafana Trace-to-Log Correlation + +**Objective**: Enable one-click navigation from Tempo traces to Loki logs in Grafana. + +**What to do**: + +- Edit Grafana Tempo data source provisioning to add `tracesToLogs` configuration: + + ```yaml + tracesToLogs: + datasourceUid: loki + filterByTraceID: true + filterBySpanID: false + tags: ["partition", "severity"] + ``` + +- Edit Grafana Loki data source provisioning to add `derivedFields` linking trace_id back to Tempo: + ```yaml + derivedFields: + - datasourceUid: tempo + matcherRegex: "trace_id=(\\w+)" + name: TraceID + url: "$${__value.raw}" + ``` + +**Key modified files**: + +- `docker/telemetry/grafana/provisioning/datasources/loki.yaml` +- `docker/telemetry/grafana/provisioning/datasources/` (Tempo data source file) + +--- + +## Task 8.5: Update Integration Tests + +**Objective**: Verify trace_id appears in logs and Loki correlation works. + +**What to do**: + +- Edit `docker/telemetry/integration-test.sh`: + - After sending RPC requests (which create spans), grep xrpld's log output for `trace_id=` + - Verify trace_id matches a trace visible in Tempo + - Optionally: query Loki via API to confirm log ingestion + +**Key modified files**: + +- `docker/telemetry/integration-test.sh` + +--- + +## Task 8.6: Update Documentation + +**Objective**: Document the log correlation feature in runbook and reference docs. + +**What to do**: + +- Edit `docs/telemetry-runbook.md`: + - Add "Log-Trace Correlation" section explaining how to use Grafana Tempo -> Loki linking + - Add LogQL query examples for filtering by trace_id + +- Edit `OpenTelemetryPlan/09-data-collection-reference.md`: + - Add new section "3. Log Correlation" between SpanMetrics and StatsD sections + - Document the log format with trace_id injection + - Document Loki as a new backend + +- Edit `docker/telemetry/TESTING.md`: + - Add log correlation verification steps + +**Key modified files**: + +- `docs/telemetry-runbook.md` +- `OpenTelemetryPlan/09-data-collection-reference.md` +- `docker/telemetry/TESTING.md` + +--- + +## Summary Table + +| Task | Description | Sub-Phase | New Files | Modified Files | Depends On | +| ---- | ------------------------------------------ | --------- | --------- | -------------- | ---------- | +| 8.1 | Inject trace_id into Logs::format() | 8a | 0 | 1 | Phase 7 | +| 8.2 | Add Loki to Docker Compose stack | 8b | 1 | 1 | -- | +| 8.3 | Add filelog receiver to OTel Collector | 8b | 0 | 2 | 8.1, 8.2 | +| 8.4 | Configure Grafana trace-to-log correlation | 8b | 0 | 2 | 8.3 | +| 8.5 | Update integration tests | 8a + 8b | 0 | 1 | 8.4 | +| 8.6 | Update documentation | 8a + 8b | 0 | 3 | 8.5 | + +**Parallel work**: Task 8.2 (Loki infra) can run in parallel with Task 8.1 (code change). Tasks 8.3-8.6 are sequential. + +**Exit Criteria** (from [06-implementation-phases.md §6.8.1](./06-implementation-phases.md)): + +- [ ] Log lines within active spans contain `trace_id= span_id=` +- [ ] Log lines outside spans have no trace context (no empty fields) +- [ ] Loki ingests xrpld logs via OTel Collector filelog receiver +- [ ] Grafana Tempo -> Loki one-click correlation works +- [ ] Grafana Loki -> Tempo reverse lookup works via derived field +- [ ] Integration test verifies trace_id presence in logs +- [ ] No performance regression from trace_id injection (< 0.1% overhead) diff --git a/OpenTelemetryPlan/Phase9_taskList.md b/OpenTelemetryPlan/Phase9_taskList.md new file mode 100644 index 00000000000..74babaef8f6 --- /dev/null +++ b/OpenTelemetryPlan/Phase9_taskList.md @@ -0,0 +1,549 @@ +# Phase 9: Internal Metric Instrumentation Gap Fill — Task List + +> **Status**: Future Enhancement +> +> **Goal**: Instrument xrpld to emit ~50+ metrics that exist in `get_counts`/`server_info`/TxQ/PerfLog but currently lack time-series export via the OTel or beast::insight pipelines. +> +> **Scope**: Hybrid approach — extend `beast::insight` for metrics near existing registrations, use OTel Metrics SDK `ObservableGauge` callbacks for new categories (TxQ, PerfLog, CountedObjects). +> +> **Branch**: `pratik/otel-phase9-metric-gap-fill` (from `pratik/otel-phase8-log-correlation`) +> +> **Depends on**: Phase 7 (native OTel metrics pipeline) and Phase 8 (log-trace correlation) + +### Related Plan Documents + +| Document | Relevance | +| -------------------------------------------------------------------- | -------------------------------------------------------------- | +| [06-implementation-phases.md](./06-implementation-phases.md) | Phase 9 plan: motivation, architecture, exit criteria (§6.8.2) | +| [09-data-collection-reference.md](./09-data-collection-reference.md) | Current metric inventory + future metrics section | +| [Phase7_taskList.md](./Phase7_taskList.md) | Prerequisite — OTel Metrics SDK and `OTelCollector` class | +| [Phase8_taskList.md](./Phase8_taskList.md) | Prerequisite — log-trace correlation | + +### Third-Party Consumer Context + +These metrics serve multiple external consumer categories identified during research: + +| Consumer Category | Key Metrics They Need | +| ------------------------- | --------------------------------------------------------------- | +| **Exchanges** | Fee escalation levels, TxQ depth, settlement latency | +| **Payment Processors** | Load factors, io_latency, transaction throughput | +| **Analytics Providers** | NodeStore I/O, cache hit rates, counted objects | +| **Validators/Operators** | Per-job execution times, PerfLog RPC counters, consensus timing | +| **Academic Researchers** | Consensus performance time-series, fee market dynamics | +| **Institutional Custody** | Server health scores, reserve calculations, node availability | + +--- + +## Task 9.1: NodeStore I/O Metrics + +**Objective**: Export node store read/write performance as time-series metrics. + +**What to do**: + +- In `src/libxrpl/nodestore/Database.cpp`, extend existing `beast::insight` registrations to add: + - Gauge: `node_reads_total` (cumulative read operations) + - Gauge: `node_reads_hit` (cache-served reads) + - Gauge: `node_writes` (cumulative write operations) + - Gauge: `node_written_bytes` (cumulative bytes written) + - Gauge: `node_read_bytes` (cumulative bytes read) + - Gauge: `node_reads_duration_us` (cumulative read time in microseconds) + - Gauge: `write_load` (current write load score) + - Gauge: `read_queue` (items in read queue) + +- These values are already computed in `Database::getCountsJson()` (line ~236). Wire the same counters to `beast::insight` hooks. + +**Key modified files**: + +- `src/libxrpl/nodestore/Database.cpp` +- `src/libxrpl/nodestore/Database.h` (add insight members) + +**Derived Prometheus metrics**: `xrpld_nodestore_reads_total`, `xrpld_nodestore_reads_hit`, `xrpld_nodestore_write_load`, etc. + +**Grafana dashboard**: Add "NodeStore I/O" panel group to _Node Health_ dashboard. + +--- + +## Task 9.2: Cache Hit Rate Metrics + +**Objective**: Export SHAMap and ledger cache performance as time-series gauges. + +**What to do**: + +- Register OTel `ObservableGauge` callbacks (via Phase 7's `OTelCollector`) for: + - `SLE_hit_rate` — SLE cache hit rate (0.0–1.0) + - `ledger_hit_rate` — Ledger object cache hit rate + - `AL_hit_rate` — AcceptedLedger cache hit rate + - `treenode_cache_size` — SHAMap TreeNode cache size (entries) + - `treenode_track_size` — Tracked tree nodes + - `fullbelow_size` — FullBelow cache size + +- The callback should read from the same sources as `GetCounts.cpp` handler (line ~43). + +- Create a centralized `MetricsRegistry` class that holds all OTel async gauge registrations, polled at 10-second intervals by the `PeriodicMetricReader`. + +**Key modified files**: + +- New: `src/xrpld/telemetry/MetricsRegistry.h` / `.cpp` +- `src/xrpld/rpc/handlers/GetCounts.cpp` (extract shared access methods) +- `src/xrpld/app/main/Application.cpp` (register MetricsRegistry at startup) + +**Derived Prometheus metrics**: `xrpld_cache_SLE_hit_rate`, `xrpld_cache_ledger_hit_rate`, `xrpld_cache_treenode_size`, etc. + +--- + +## Task 9.3: Transaction Queue (TxQ) Metrics + +**Objective**: Export TxQ depth, capacity, and fee escalation levels as time-series. + +**What to do**: + +- Register OTel `ObservableGauge` callbacks for TxQ state (from `TxQ.h` line ~143): + - `txq_count` — Current transactions in queue + - `txq_max_size` — Maximum queue capacity + - `txq_in_ledger` — Transactions in current open ledger + - `txq_per_ledger` — Expected transactions per ledger + - `txq_reference_fee_level` — Reference fee level + - `txq_min_processing_fee_level` — Minimum fee to get processed + - `txq_med_fee_level` — Median fee level in queue + - `txq_open_ledger_fee_level` — Open ledger fee escalation level + +- Add to the `MetricsRegistry` (Task 9.2). + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.cpp` (add TxQ callbacks) +- `src/xrpld/app/tx/detail/TxQ.h` (expose metrics accessor if needed) + +**Derived Prometheus metrics**: `xrpld_txq_count`, `xrpld_txq_max_size`, `xrpld_txq_open_ledger_fee_level`, etc. + +**Grafana dashboard**: New _Fee Market & TxQ_ dashboard (`fee-market`). + +--- + +## Task 9.4: PerfLog Per-RPC Method Metrics + +**Objective**: Export per-RPC-method call counts and latency as OTel metrics. + +**What to do**: + +- Register OTel instruments for PerfLog RPC counters (from `PerfLogImp.cpp` line ~63): + - Counter: `xrpld_rpc_method_started_total{method=""}` — calls started + - Counter: `xrpld_rpc_method_finished_total{method=""}` — calls completed + - Counter: `xrpld_rpc_method_errored_total{method=""}` — calls errored + - Histogram: `xrpld_rpc_method_duration_us{method=""}` — execution time distribution + +- Use OTel `Counter` and `Histogram` instruments with `method` attribute label. + +- Hook into the existing PerfLog callback mechanism rather than adding new instrumentation points. + +**Key modified files**: + +- `src/xrpld/perflog/detail/PerfLogImp.cpp` (add OTel instrument updates alongside existing JSON counters) +- `src/xrpld/telemetry/MetricsRegistry.cpp` (register instruments) + +**Derived Prometheus metrics**: `xrpld_rpc_method_started_total{method="server_info"}`, `xrpld_rpc_method_duration_us_bucket{method="ledger"}`, etc. + +**Grafana dashboard**: Add "Per-Method RPC Breakdown" panel group to _RPC Performance_ dashboard. + +--- + +## Task 9.5: PerfLog Per-Job-Type Metrics + +**Objective**: Export per-job-type queue and execution metrics. + +**What to do**: + +- Register OTel instruments for PerfLog job counters: + - Counter: `xrpld_job_queued_total{job_type=""}` — jobs queued + - Counter: `xrpld_job_started_total{job_type=""}` — jobs started + - Counter: `xrpld_job_finished_total{job_type=""}` — jobs completed + - Histogram: `xrpld_job_queued_duration_us{job_type=""}` — time spent waiting in queue + - Histogram: `xrpld_job_running_duration_us{job_type=""}` — execution time distribution + +- Hook into PerfLog's existing job tracking alongside Task 9.4. + +**Key modified files**: + +- `src/xrpld/perflog/detail/PerfLogImp.cpp` +- `src/xrpld/telemetry/MetricsRegistry.cpp` + +**Derived Prometheus metrics**: `xrpld_job_queued_total{job_type="ledgerData"}`, `xrpld_job_running_duration_us_bucket{job_type="transaction"}`, etc. + +**Grafana dashboard**: New _Job Queue Analysis_ dashboard (`job-queue`). + +--- + +## Task 9.6: Counted Object Instance Metrics + +**Objective**: Export live instance counts for key internal object types. + +**What to do**: + +- Register OTel `ObservableGauge` callbacks for `CountedObject` instance counts: + - `xrpld_object_count{type="Transaction"}` — live Transaction objects + - `xrpld_object_count{type="Ledger"}` — live Ledger objects + - `xrpld_object_count{type="NodeObject"}` — live NodeObject instances + - `xrpld_object_count{type="STTx"}` — serialized transaction objects + - `xrpld_object_count{type="STLedgerEntry"}` — serialized ledger entries + - `xrpld_object_count{type="InboundLedger"}` — ledgers being fetched + - `xrpld_object_count{type="Pathfinder"}` — active pathfinding computations + - `xrpld_object_count{type="PathRequest"}` — active path requests + - `xrpld_object_count{type="HashRouterEntry"}` — hash router entries + +- The `CountedObject` template already tracks these via atomic counters. The callback just reads the current counts. + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.cpp` (add counted object callbacks) +- `include/xrpl/basics/CountedObject.h` (may need static accessor for iteration) + +**Derived Prometheus metrics**: `xrpld_object_count{type="Transaction"}`, `xrpld_object_count{type="NodeObject"}`, etc. + +**Grafana dashboard**: Add "Object Instance Counts" panel to _Node Health_ dashboard. + +--- + +## Task 9.7: Fee Escalation & Load Factor Metrics + +**Objective**: Export the full load factor breakdown as time-series. + +**What to do**: + +- Register OTel `ObservableGauge` callbacks for load factors (from `NetworkOPs.cpp` line ~2694): + - `load_factor` — combined transaction cost multiplier + - `load_factor_server` — server + cluster + network contribution + - `load_factor_local` — local server load only + - `load_factor_net` — network-wide load estimate + - `load_factor_cluster` — cluster peer load + - `load_factor_fee_escalation` — open ledger fee escalation + - `load_factor_fee_queue` — queue entry fee level + +- These overlap with some existing StatsD metrics but provide finer granularity (individual factor breakdown vs. combined value). + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.cpp` +- `src/xrpld/app/misc/NetworkOPs.cpp` (expose load factor accessors if needed) + +**Derived Prometheus metrics**: `xrpld_load_factor`, `xrpld_load_factor_fee_escalation`, etc. + +**Grafana dashboard**: Add "Load Factor Breakdown" panel to _Fee Market & TxQ_ dashboard. + +--- + +## Task 9.7a: push_metrics.py Parity — Missing Observable Gauges + +**Objective**: Fill the remaining metric gaps between the external `push_metrics.py` script (in `ripplex-ansible`) and the internal OTel `MetricsRegistry` observable gauges. After this task, all metrics collected by `push_metrics.py` that CAN be collected internally are covered. + +**What was done**: + +- Extended existing `cacheHitRateGauge_` callback with `AL_size` (AcceptedLedger cache size) +- Extended existing `nodeStoreGauge_` callback with 4 new metrics from `getCountsJson()`: + - `node_reads_duration_us` (JSON string — uses `std::stoll(asString())`) + - `read_request_bundle` (native JSON int) + - `read_threads_running` (native JSON int) + - `read_threads_total` (native JSON int) +- Added new `xrpld_server_info` Int64ObservableGauge with 8 metrics: + - `server_state` — operating mode as int (0=DISCONNECTED .. 4=FULL) + - `uptime` — seconds since server start + - `peers` — total peer count + - `validated_ledger_seq` — validated ledger sequence (atomic read) + - `ledger_current_index` — current open ledger sequence + - `peer_disconnects_resources` — cumulative resource-related disconnects + - `last_close_proposers` — from `getConsensusInfo()["previous_proposers"]` + - `last_close_converge_time_ms` — from `getConsensusInfo()["previous_mseconds"]` +- Added new `xrpld_build_info` Int64ObservableGauge (info-style, value=1 with `version` label) +- Added new `xrpld_complete_ledgers` Int64ObservableGauge parsing comma-separated ranges into `{bound, index}` pairs +- Added new `xrpld_db_metrics` Int64ObservableGauge with 4 metrics: + - `db_kb_total`, `db_kb_ledger`, `db_kb_transaction` (SQLite stat queries) + - `historical_perminute` (historical ledger fetch rate) + +**Key modified files**: + +- `src/xrpld/telemetry/MetricsRegistry.h` (4 new gauge members, updated ASCII diagram) +- `src/xrpld/telemetry/MetricsRegistry.cpp` (4 new callback registrations, 2 callback extensions) + +**Not implementable inside xrpld**: + +- `connection_count_51233/51234` — OS-level port connection counts from external shell script (`get_connection.sh`) + +**Derived Prometheus metrics**: `xrpld_server_info{metric="server_state"}`, `xrpld_build_info{version="2.4.0"}`, `xrpld_complete_ledgers{bound="start",index="0"}`, `xrpld_db_metrics{metric="db_kb_total"}`, etc. + +**Grafana dashboard**: New panels added to _Node Health_ dashboard (`node-health.json`). + +--- + +## Task 9.8: New Grafana Dashboards + +**Objective**: Create Grafana dashboards for the new metric categories. + +**What to do**: + +- Create 2 new dashboards: + 1. **Fee Market & TxQ** (`fee-market`) — TxQ depth/capacity, fee levels, load factor breakdown, fee escalation timeline + 2. **Job Queue Analysis** (`job-queue`) — Per-job-type rates, queue wait times, execution times, job queue depth + +- Update 2 existing dashboards: + 1. **Node Health** (`xrpld-statsd-node-health`) — Add NodeStore I/O panels, cache hit rate panels, object instance counts + 2. **RPC Performance** (`rpc-performance`) — Add per-method RPC breakdown panels + +**Key modified files**: + +- New: `docker/telemetry/grafana/dashboards/rippled-fee-market.json` +- New: `docker/telemetry/grafana/dashboards/rippled-job-queue.json` +- `docker/telemetry/grafana/dashboards/rippled-statsd-node-health.json` +- `docker/telemetry/grafana/dashboards/rippled-rpc-perf.json` + +--- + +## Task 9.9: Update Documentation + +**Objective**: Update telemetry reference docs with all new metrics. + +**What to do**: + +- Update `OpenTelemetryPlan/09-data-collection-reference.md`: + - Add new section for OTel SDK-exported metrics (NodeStore, cache, TxQ, PerfLog, CountedObjects, load factors) + - Update Grafana dashboard reference table (add 2 new dashboards) + - Add Prometheus query examples for new metrics + +- Update `docs/telemetry-runbook.md`: + - Add an Alerting section covering the provisioned rules and how to wire a receiver + - Add troubleshooting entries for new metric categories + +- Provision Grafana alert rules (`docker/telemetry/grafana/provisioning/alerting/`): + - 6 rules in 3 groups — consensus/ledger (`LedgerHistoryMismatch`, `LedgerCloseStalled`), validator (`ValidationsMissed`, `ValidationsNotChecked`), job queue (`JobQueueTxOverflow`, `JobQueueLatencyHigh`) + - `xrpld-default` webhook contact point + flat notification policy; auto-loaded via the existing `provisioning/` mount (no docker-compose change) + - Alerting operator docs (per-alert meaning, tuning, receiver wiring) now live in the Alerting section of `docs/telemetry-runbook.md` + +**Key modified files**: + +- `OpenTelemetryPlan/09-data-collection-reference.md` +- `docs/telemetry-runbook.md` +- `docker/telemetry/grafana/provisioning/alerting/{rules,contactpoints,policies}.yaml` (new) +- `docs/telemetry-runbook.md` (Alerting section added) + +--- + +## Task 9.10: Integration Tests + +**Objective**: Verify all new metrics appear in Prometheus after a test workload. + +**What to do**: + +- Extend the existing telemetry integration test: + - Start xrpld with `[telemetry] enabled=1` and `[insight] server=otel` + - Submit a batch of RPC calls and transactions + - Query Prometheus for each new metric family + - Assert non-zero values for: NodeStore reads, cache hit rates, TxQ count, PerfLog RPC counters, object counts, load factors + +- Add unit tests for the `MetricsRegistry` class: + - Verify callback registration and deregistration + - Verify metric values match `get_counts` JSON output + - Verify graceful behavior when telemetry is disabled + +**Key modified files**: + +- `src/test/telemetry/MetricsRegistry_test.cpp` (new) +- Existing integration test script (extend assertions) + +--- + +## Task 9.11: Validator Health Dashboard (External Dashboard Parity) + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — dashboards for Phase 7 metrics inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard). +> +> **Upstream**: Phase 7 Tasks 7.9-7.16 (metrics must be emitting). +> **Downstream**: Phase 10 (dashboard load checks), Phase 11 (alert rules reference these panels). + +**Objective**: Create a Grafana dashboard for validation agreement, amendment/UNL health, and state tracking. + +**Dashboard**: `validator-health.json` + +| Panel | Type | PromQL | +| -------------------------- | ---------- | -------------------------------------------------------------- | +| Agreement % (1h) | stat | `xrpld_validation_agreement{metric="agreement_pct_1h"}` | +| Agreement % (24h) | stat | `xrpld_validation_agreement{metric="agreement_pct_24h"}` | +| Agreements vs Missed (1h) | bargauge | `agreements_1h` and `missed_1h` side by side | +| Agreements vs Missed (24h) | bargauge | `agreements_24h` and `missed_24h` side by side | +| Validation Rate | stat | `rate(xrpld_validations_sent_total[5m]) * 60` | +| Validations Checked Rate | stat | `rate(xrpld_validations_checked_total[5m]) * 60` | +| Amendment Blocked | stat | `xrpld_validator_health{metric="amendment_blocked"}` | +| UNL Expiry (days) | stat | `xrpld_validator_health{metric="unl_expiry_days"}` | +| Validation Quorum | stat | `xrpld_validator_health{metric="validation_quorum"}` | +| State Value Timeline | timeseries | `xrpld_state_tracking{metric="state_value"}` | +| Time in Current State | stat | `xrpld_state_tracking{metric="time_in_current_state_seconds"}` | +| State Changes Rate | stat | `rate(xrpld_state_changes_total[1h])` | +| Ledgers Closed Rate | stat | `rate(xrpld_ledgers_closed_total[5m]) * 60` | + +**Dashboard conventions**: `$node` template variable for `service_instance_id` filtering, dark theme, matching existing panel sizes and color schemes. + +**Key new files**: `docker/telemetry/grafana/dashboards/rippled-validator-health.json` + +**Exit Criteria**: + +- [ ] All 13 panels render with non-zero data during normal operation +- [ ] `$node` filter works correctly for multi-node deployments +- [ ] Amendment blocked and UNL expiry panels use color thresholds (red=blocked/expiring) + +--- + +## Task 9.12: Peer Quality Dashboard (External Dashboard Parity) + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Create a Grafana dashboard for peer health aggregates. + +**Dashboard**: `peer-quality.json` + +| Panel | Type | PromQL | +| ---------------------- | ---------- | -------------------------------------------------------------- | +| P90 Peer Latency | timeseries | `xrpld_peer_quality{metric="peer_latency_p90_ms"}` | +| Insane/Diverged Peers | stat | `xrpld_peer_quality{metric="peers_insane_count"}` | +| Higher Version Peers % | stat | `xrpld_peer_quality{metric="peers_higher_version_pct"}` | +| Upgrade Recommended | stat | `xrpld_peer_quality{metric="upgrade_recommended"}` | +| Resource Disconnects | timeseries | `xrpld_Overlay_Peer_Disconnects_Charges` | +| Inbound vs Outbound | bargauge | `xrpld_Peer_Finder_Active_Inbound_Peers`, `..._Outbound_Peers` | + +**Key new files**: `docker/telemetry/grafana/dashboards/rippled-peer-quality.json` + +**Exit Criteria**: + +- [ ] All 6 panels render correctly +- [ ] P90 latency panel shows trend over time +- [ ] Upgrade recommended panel uses color threshold (red=1, green=0) + +--- + +## Task 9.13: Ledger Economy Dashboard Panels (External Dashboard Parity) + +> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) + +**Objective**: Add "Ledger Economy" row to the existing `node-health.json` dashboard. + +| Panel | Type | PromQL | +| -------------------- | ---------- | --------------------------------------------------- | +| Base Fee (drops) | stat | `xrpld_ledger_economy{metric="base_fee_xrp"}` | +| Reserve Base (drops) | stat | `xrpld_ledger_economy{metric="reserve_base_xrp"}` | +| Reserve Inc (drops) | stat | `xrpld_ledger_economy{metric="reserve_inc_xrp"}` | +| Ledger Age | stat | `xrpld_ledger_economy{metric="ledger_age_seconds"}` | +| Transaction Rate | timeseries | `xrpld_ledger_economy{metric="transaction_rate"}` | + +**Key modified files**: `docker/telemetry/grafana/dashboards/node-health.json` + +**Exit Criteria**: + +- [ ] 5 new panels render correctly in existing dashboard +- [ ] Fee values match `server_info` RPC output +- [ ] Transaction rate shows smooth trend (not spiky) + +--- + +## Exit Criteria + +- [ ] All ~50 new metrics visible in Prometheus via OTLP pipeline +- [ ] `MetricsRegistry` class registers/deregisters cleanly with OTel SDK +- [ ] Async gauge callbacks execute at 10s intervals without performance impact +- [ ] 2 new Grafana dashboards operational (Fee Market, Job Queue) +- [ ] 2 existing dashboards updated with new panel groups +- [ ] Integration test validates all new metric families are non-zero +- [ ] No performance regression (< 0.5% CPU overhead from new callbacks) +- [ ] Documentation updated with full new metric inventory +- [ ] Validator Health dashboard renders all 13 panels +- [ ] Peer Quality dashboard renders all 6 panels +- [ ] Ledger Economy panels added to node-health dashboard + +--- + +## Appendix: Alerting Design + +> Design for the provisioned Grafana alert rules (Task 9.9a). Previously a standalone spec; merged here so the phase plan is self-contained. + +**Date:** 2026-07-06 +**Branch:** `pratik/otel-phase9-metric-gap-fill` (PR #6513, Jira RIPD-5187) +**Status:** Approved + +### Purpose + +Phase 9 exports ~68 internal xrpld metrics and ships Grafana dashboards for +them. This adds the missing operator-facing piece: **provisioned Grafana alert +rules** that fire on the health-critical metrics phase 9 introduces. The +phase-9 task list (line 311) and Jira story RIPD-5187 both already list +"alerting rules" as a phase-9 deliverable, so this closes that gap. + +Scope is deliberately narrow — the three subsystems whose failure is +node-fatal: **consensus/ledger health, validator health, job queue**. RPC/API +health is explicitly out of scope. + +### Why phase 9 (not phase 11) + +Every metric these alerts fire on is _born_ in phase 9 +(`xrpld_ledger_history_mismatch_total`, `xrpld_ledgers_closed_total`, +`xrpld_validation_missed_total`, `xrpld_validations_checked_total`, +`xrpld_jq_trans_overflow_total`, `xrpld_job_queued_duration_us_bucket`). Alerts +belong with the metrics they watch, and this is where the dependency lives. + +### Delivery + +Provisioned YAML, version-controlled — matching the existing datasource / +dashboard provisioning pattern. No docker-compose change: the Grafana service +already mounts `./grafana/provisioning:/etc/grafana/provisioning:ro`, and +Grafana auto-loads `provisioning/alerting/*.yaml`. + +New files under `docker/telemetry/grafana/provisioning/alerting/`: + +| File | Purpose | +| -------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `contactpoints.yaml` | One contact point `xrpld-default` (webhook to a documented placeholder; comments show how to swap for Slack/email). | +| `policies.yaml` | Default notification policy: route all alerts → `xrpld-default`, grouped by `alertname` + `service_instance_id`. | +| `rules.yaml` | 6 alert rules across 3 groups (below). | + +Plus the Alerting section of `docs/telemetry-runbook.md` — operator runbook: +what each alert means, likely causes, and how to point the contact point at a +real receiver. + +### Alert rules + +All rules target Prometheus datasource `uid: prometheus`. Each rule uses the +Grafana rule shape: query (A) → reduce (B, last value) → threshold (C). All +`rate()`/`histogram_quantile()` expressions aggregate with +`sum by (service_instance_id)` (or `+ le`) so **each node alerts independently**. +Alert rules run headless, so they cannot use the dashboards' `$node` template +variables — they match all series and group by `service_instance_id` instead. + +| Group | Alert | Expression (5m window) | Fires | `for` | severity | +| --------- | --------------------- | ----------------------------------------------------------------------------------------------------------- | --------------------- | ----- | -------- | +| Consensus | LedgerHistoryMismatch | `sum by (service_instance_id)(rate(xrpld_ledger_history_mismatch_total[5m]))` | `> 0` | 5m | critical | +| Consensus | LedgerCloseStalled | `sum by (service_instance_id)(rate(xrpld_ledgers_closed_total[5m]))` | `< 0.001` (≈0) | 3m | critical | +| Validator | ValidationsMissed | `sum by (service_instance_id)(rate(xrpld_validation_missed_total[5m]))` | `> 0` | 5m | warning | +| Validator | ValidationsNotChecked | `sum by (service_instance_id)(rate(xrpld_validations_checked_total[5m]))` | `< 0.001` (≈0) | 5m | warning | +| Job queue | JobQueueTxOverflow | `sum by (service_instance_id)(rate(xrpld_jq_trans_overflow_total[5m]))` | `> 0` | 5m | warning | +| Job queue | JobQueueLatencyHigh | `histogram_quantile(0.99, sum by (le, service_instance_id)(rate(xrpld_job_queued_duration_us_bucket[5m])))` | `> 1000000` (µs = 1s) | 5m | warning | + +Each rule carries labels `severity` and `category` (consensus/validator/jobqueue) +and annotations `summary` + `description` (with `{{ $labels.service_instance_id }}` +and `{{ $values.B.Value }}` interpolation). + +#### Threshold rationale + +- **LedgerCloseStalled `< 0.001` for 3m**: healthy nodes close a ledger every + ~3-5s; a 5m rate decaying to ~0 means the node is stuck. The epsilon (not + exact `0`) avoids float rate-noise suppressing the alert. +- **JobQueueLatencyHigh 1s p99**: a default starting point, easy to tune — jobs + queued >1s at p99 indicate the node is saturated. +- Others are `> 0` on error/miss counters: any sustained nonzero rate is + actionable. + +### Non-goals / YAGNI + +- No per-alert silencing schedules, no mute timings. +- No RPC/API, overlay, or fee-market alerts (dashboards cover those visually). +- Single contact point — multi-receiver routing is left to the operator. + +### Verification + +1. `yamllint` (or `python -c yaml.safe_load`) on all three YAML files. +2. `docker compose -f docker/telemetry/docker-compose.yml config -q` still parses. +3. Optional live check: start stack, `GET /api/v1/provisioning/alert-rules` + returns the 6 rules; Grafana logs show no provisioning errors. +4. Code-review pass (subagent) against phase conventions before commit. diff --git a/OpenTelemetryPlan/secure-OTel.md b/OpenTelemetryPlan/secure-OTel.md new file mode 100644 index 00000000000..3d22de5452f --- /dev/null +++ b/OpenTelemetryPlan/secure-OTel.md @@ -0,0 +1,240 @@ +# Securing OpenTelemetry Against Trace Context Spoofing + +> **Part of**: [OpenTelemetry Implementation Plan](./OpenTelemetryPlan.md) — see also [Design Decisions § Privacy](./02-design-decisions.md#244-privacy--sensitive-data-policy) (what we don't collect) and [Configuration Reference § 5.5](./05-configuration-reference.md#55-opentelemetry-collector-configuration) (collector base config). + +Trace context spoofing (or poisoning) occurs when untrusted actors inject tampered or stale trace IDs into your system. If these requests are processed, the spans are appended to historical trace buckets, stretching trace durations, ruining p99 latency metrics, and breaking Grafana dashboards. + +This guide outlines two categories of defense: mitigating tampered contexts and locking down the OpenTelemetry (OTel) Collector to trusted clients only. + +--- + +## Part 1: Mitigating Tampered Trace Contexts + +### 1. Perimeter Defense: Strip Headers at the API Gateway + +The most effective way to prevent spoofing from external sources is to treat your API Gateway (Envoy, NGINX, AWS ALB) as a hard boundary. Strip incoming W3C tracing headers (`traceparent`, `tracestate`) from public traffic so the gateway is forced to generate a fresh, legitimate `trace_id`. + +**NGINX Example (Stripping Headers):** + +```nginx +server { + listen 80; + + location / { + # Clear out untrusted incoming trace headers + proxy_set_header traceparent ""; + proxy_set_header tracestate ""; + + proxy_pass http://backend_service; + } +} +``` + +### **2. Timestamp-Anchored Trace IDs and OTTL Filtering** + +If you use a custom trace ID generator that embeds a timestamp in the first few bytes (like AWS X-Ray or UUIDv7), you can use the OTel Collector's OpenTelemetry Transform Language (OTTL) to detect anomalies. +**Collector Configuration (Conceptual OTTL Filter):** + +```yaml +processors: + filter/stale_traces: + error_mode: ignore + traces: + span: + # Example: Drop spans where the start time is significantly different + # from an expected parameter or embedded timestamp logic. + # Note: Standard W3C trace IDs do not contain timestamps by default. + - 'Keep out-of-bounds spans: time.sub(start_time, now()) > duration("1h")' +``` + +## **Part 2: Restricting Access to the OTel Collector** + +Locking down the Collector ensures that only authenticated, trusted clients can submit telemetry data. + +### **Approach A: Network Layer Security (Kubernetes Network Policies)** + +Ensure your Collector is not exposed to the public internet. If running in Kubernetes, use a NetworkPolicy to restrict ingress traffic to specific namespaces. +**Kubernetes NetworkPolicy Example:** + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-internal-otel + namespace: observability +spec: + podSelector: + matchLabels: + app: opentelemetry-collector + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + environment: production + ports: + - protocol: TCP + port: 4317 # gRPC + - protocol: TCP + port: 4318 # HTTP +``` + +### **Approach B: Transport Layer Security (Mutual TLS / mTLS)** + +Require clients to present a valid cryptographic certificate to connect to the Collector. +**Collector Configuration (mTLS):** + +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + tls: + # Setting client_ca_file makes the collector require and verify a + # client cert, rejecting connections without a trusted one. + client_ca_file: /certs/client_ca.pem # CA that signs trusted client certs + cert_file: /certs/collector.pem + key_file: /certs/collector.key +``` + +### **Approach C: Application Layer Authentication (Basic Auth Extension)** + +Use the Collector's extension system to require an API key or Basic Auth credentials. +**Collector Configuration (Basic Auth):** + +```yaml +extensions: + basicauth/collector: + htpasswd: + inline: | + # username:trusted-client, password:SecurePassword123 + trusted-client:$apr1$4v8p76o6$DMTX5Wv6uOmrFAZp2X1N1. + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + auth: + authenticator: basicauth/collector + +processors: + batch: + +exporters: + otlp: + endpoint: my-backend-storage:4317 + +service: + extensions: [basicauth/collector] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp] +``` + +**Client Setup (Environment Variables):** +Developers must pass the authentication header using the standard OTel SDK environment variables: + +```bash +# Base64 encoded "trusted-client:SecurePassword123" +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic dHJ1c3RlZC1jbGllbnQ6U2VjdXJlUGFzc3dvcmQxMjM=" +``` + +--- + +Available routes to build on top of: https://github.com/XRPLF/rippled/pull/6425#discussion_r3234751995 + +--- + +# Analysis: Applying the Guide to xrpld + +The guide above is written for HTTP-fronted web services. xrpld is a P2P node daemon, so the threat model and the applicable defenses differ. This section captures how each approach maps to xrpld and the chosen direction. + +## Threat Model + +xrpld has **two distinct attack surfaces**, not one. The original guide conflates them under "trace context spoofing"; for xrpld they need separate defenses. + +| Surface | Attacker | Vector | Defense | +| ----------------------------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------- | +| **Collector ingress** (xrpld → collector) | Anyone who can reach `4317`/`4318` on the collector host | Forged OTLP traffic, telemetry exfiltration, DoS on collector | mTLS + network policy | +| **Peer trace context** (peer → xrpld) | Malicious peer in the XRPL overlay | Crafted `protocol::TraceContext` field inside peer protobuf messages (TMTransaction, consensus, etc.) — used to forge `trace_id`/`span_id`, pollute p99, attach spans to historical traces | Validate + rate-limit at the receive boundary | + +**Deployment context:** Across-network. xrpld nodes (potentially run by external operators or in different DCs) ship telemetry to a centrally-hosted collector across an untrusted network. The collector is NOT on the same host or private VPC as every node. + +``` + ┌── peer (untrusted) ── TMTransaction{trace_context} ──▶ xrpld + │ │ + │ [validate + rate-limit] + │ │ + │ ▼ + │ SpanGuard (clean) + │ │ + │ │ OTLP/gRPC + │ │ + mTLS + │ ▼ + └───────────────────────────────────────── [client_ca_file: verify client cert] + OTel Collector + (in private subnet, NetPol) +``` + +## Part 1 Applicability — Peer Trace-Context Validation + +The guide's NGINX header stripping and OTTL stale-span filtering target HTTP gateways and post-hoc cleanup. Neither fits xrpld directly: + +- **NGINX header stripping** — N/A. There is no HTTP gateway between peers and xrpld; trace context arrives inside protobuf peer messages (`protocol::TraceContext`), not as W3C `traceparent` headers. See [src/xrpld/telemetry/PropagationHelpers.h](../src/xrpld/telemetry/PropagationHelpers.h). +- **OTTL stale-span filtering** — Weak fit. Post-hoc cleanup at the collector loses peer identity (you can't tell _which_ peer poisoned the trace). Validation at the receive site is stronger. + +**xrpld-specific Part 1 mitigations:** + +1. **Validate extracted context at the boundary** in [src/xrpld/telemetry/ConsensusReceiveTracing.h](../src/xrpld/telemetry/ConsensusReceiveTracing.h) and any other peer-message receive site. Reject if `trace_id` is all-zero, wrong length, or fails W3C format checks. Treat invalid context as "no propagated context" — start a fresh span — rather than dropping the message. +2. **Per-peer sample rate limiting** so a hostile peer cannot flood the collector with spans bearing a fabricated `trace_id`. Use probabilistic sampling on the receive path keyed by peer identity. + +## Part 2 — Comparison of Collector Hardening Approaches + +Evaluated for the across-network deployment shape: + +| Approach | Across-network fit | Cost | Verdict | +| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | ---------------------------------- | +| **A. NetworkPolicy / firewall** | Necessary baseline (don't expose `4317`/`4318` to the internet), but insufficient on its own when traffic genuinely crosses networks — you cannot NetworkPolicy the public internet. | Cheap. | **Defense-in-depth, not primary.** | +| **B. mTLS** | Strongest fit. Every xrpld node holds a client cert; the collector verifies it via `client_ca_file` in the receiver's `tls` block. Encrypts in transit (raw OTLP over the internet leaks transaction patterns and validator identity). Compromised node = revoke one cert, no shared secret to rotate everywhere. | Cert issuance + rotation pipeline. | **Primary.** | +| **C. Basic Auth** | Worst shape for this topology. Single shared password across all xrpld nodes — one leaked node config compromises the whole fleet. Doesn't encrypt; you'd need TLS underneath anyway, at which point you're 80% of the way to mTLS. | Cheap to set up, expensive to operate (rotation across N operators). | **Skip.** | + +## Decision + +**Primary defense:** mTLS (Approach B) on the collector's OTLP receivers. The collector requires and verifies each client certificate when `client_ca_file` is set in the receiver's `tls` block (there is no `auth_type` field — setting `client_ca_file` is what enforces client-cert verification). + +**Defense-in-depth:** NetworkPolicy / firewall rules (Approach A) so `4317`/`4318` are never reachable from outside the expected operator subnets even if mTLS were misconfigured. + +**Skipped:** Basic Auth (Approach C) — wrong shape for an across-network, multi-operator topology. + +**Plus xrpld-specific Part 1 work:** trace-context validation and per-peer rate limiting at peer-message receive sites. + +## Decisions Made + +| Decision | Choice | Rationale | +| -------------------- | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Cert source for mTLS | **Reuse XRPL node identity key** | One identity per node, no separate PKI to operate. Fits XRPL's existing trust model; requires small CA tooling step to derive/sign the OTel client cert from the node key. | +| Part 1 scope | **Include in this spec** | Collector hardening and peer trace-context validation share one threat model. Coherent design doc; can still be split into multiple PRs at implementation. | +| Dev impact | **Production-only** | Local `docker/telemetry/docker-compose.yml` keeps `insecure: true` and no auth for fast iteration. Only production deployment manifests gain mTLS. Accepted risk: minor dev/prod drift, mitigated by integration tests against a TLS-enabled collector in CI. | + +## Out of Scope + +- NGINX/Envoy header stripping (no HTTP gateway in front of xrpld-to-collector traffic). +- OTTL stale-span filtering at the collector (weaker than source validation; loses peer identity). +- Local development docker-compose hardening. +- Telemetry backend (Tempo) hardening — separate concern, downstream of the collector. + +## Next Step + +Write this up as a design doc with full sections covering: + +1. Threat model & architecture (this section, expanded) +2. Collector hardening — mTLS config, NetworkPolicy +3. Cert pipeline — deriving OTel client cert from XRPL node key +4. Peer trace-context validation — receive-site checks in `ConsensusReceiveTracing.h` +5. Per-peer span rate limiting +6. Testing & rollout diff --git a/cfg/xrpld-example.cfg b/cfg/xrpld-example.cfg index 9e334e6f4f3..54d3936c423 100644 --- a/cfg/xrpld-example.cfg +++ b/cfg/xrpld-example.cfg @@ -1621,3 +1621,108 @@ validators.txt # set to ssl_verify to 0. [ssl_verify] 1 +#------------------------------------------------------------------------------- +# +# 11. Telemetry (OpenTelemetry Tracing) +# +#------------------------------------------------------------------------------- +# +# Enables distributed tracing via OpenTelemetry. Requires building with +# -DXRPL_ENABLE_TELEMETRY=ON (telemetry Conan option). +# +# [telemetry] +# +# enabled=0 +# +# Enable or disable telemetry at runtime. Default: 0 (disabled). +# +# service_name=xrpld +# +# OTel resource attribute `service.name`. Default: xrpld. +# The node's network ID (from [network_id]) is automatically added +# as the `xrpl.network.id` and `xrpl.network.type` resource attributes. +# +# service_instance_id= +# +# OTel resource attribute `service.instance.id`. Uniquely identifies +# this node. Default: the node's public key (auto-detected). +# +# endpoint=http://localhost:4318/v1/traces +# +# The OTLP/HTTP exporter endpoint. The server sends trace data as +# protobuf-encoded HTTP POST requests to this URL. +# Default: http://localhost:4318/v1/traces. +# +# --- TLS settings for the OTLP exporter connection --- +# +# use_tls=0 +# +# Enable TLS for the OTLP/HTTP exporter connection. Default: 0 (off). +# +# tls_ca_cert= +# +# Path to a PEM-encoded CA certificate bundle for TLS verification. +# Only used when use_tls=1. Default: empty (system CA store). +# +# tls_client_cert= +# +# Path to this node's PEM-encoded client certificate, presented to the +# collector for mutual TLS (mTLS). Only used when use_tls=1. Leave empty +# for one-way (server-only) TLS. Default: empty. +# +# tls_client_key= +# +# Path to the PEM-encoded private key for tls_client_cert. Required +# whenever tls_client_cert is set. Only used when use_tls=1. +# Default: empty. +# +# Head sampling is intentionally fixed at 1.0 (sample everything) and is +# not configurable. A per-node sampling ratio would let nodes make +# divergent keep/drop decisions for the same distributed trace, producing +# broken/partial traces. A ParentBasedSampler ensures spans inheriting a +# remote parent honor the upstream decision. Reduce volume at the collector +# via tail sampling instead; for node-local post-hoc dropping use +# SpanGuard::discard() in code. +# +# trace_rpc=1 +# +# Enable tracing for JSON-RPC and WebSocket API request handling — +# command parsing, execution, and response serialization. Default: 1. +# +# trace_transactions=1 +# +# Enable tracing for the transaction lifecycle — submission, validation, +# application to ledgers, and final disposition. Default: 1. +# +# trace_consensus=1 +# +# Enable tracing for the consensus round lifecycle — proposals, +# validations, mode changes, and ledger acceptance. Default: 1. +# +# trace_peer=1 +# +# Enable tracing for peer-to-peer protocol messages — overlay message +# send/receive, peer handshakes, and routing. High volume; enabled +# by default. Default: 1. +# +# trace_ledger=1 +# +# Enable tracing for ledger close and accept operations — ledger +# building, state hashing, and write-back to the node store. Default: 1. +# +# --- Batch processor tuning --- +# +# batch_size=512 +# +# Maximum number of spans exported in a single batch. Default: 512. +# +# batch_delay_ms=5000 +# +# Maximum delay (milliseconds) before a partial batch is flushed. +# Default: 5000 (5 seconds). +# +# max_queue_size=2048 +# +# Maximum number of spans queued in memory before drops occur. +# Default: 2048. +# diff --git a/cmake/XrplCore.cmake b/cmake/XrplCore.cmake index 3e492677152..40e7c0f8542 100644 --- a/cmake/XrplCore.cmake +++ b/cmake/XrplCore.cmake @@ -78,6 +78,19 @@ include(target_link_modules) # Level 01 add_module(xrpl beast) target_link_libraries(xrpl.libxrpl.beast PUBLIC xrpl.imports.main) +# OTelCollector in beast/insight uses the OTel Metrics SDK when telemetry is +# enabled. Link the Conan-provided umbrella target rather than individual +# component targets: the OTel package's per-component dependency graph is +# under-declared (e.g. the OTLP client references sdk::common symbols without +# declaring the edge), so naming components directly reorders the static-link +# line into an unresolvable state. The umbrella carries the full, internally +# consistent graph the package authors validated. +if(telemetry) + target_link_libraries( + xrpl.libxrpl.beast + PUBLIC opentelemetry-cpp::opentelemetry-cpp + ) +endif() include(GitInfo) add_module(xrpl git) @@ -198,8 +211,36 @@ target_link_libraries( xrpl.libxrpl.conditions ) +# Telemetry module — OpenTelemetry distributed tracing support. +# Sources: include/xrpl/telemetry/ (headers), src/libxrpl/telemetry/ (impl). +# When telemetry=ON, links the Conan-provided umbrella target +# opentelemetry-cpp::opentelemetry-cpp (individual component targets like +# ::api, ::sdk are not available in the Conan package). +# +# Links xrpl.libxrpl.protocol PRIVATELY for sha512Half (digest.h) +add_module(xrpl telemetry) +target_link_libraries( + xrpl.libxrpl.telemetry + PUBLIC xrpl.libxrpl.basics xrpl.libxrpl.beast xrpl.libxrpl.config + PRIVATE xrpl.libxrpl.protocol +) +if(telemetry) + # Telemetry owns both the trace and (as of the direct-metrics API) the + # metrics pipeline. Link the umbrella target: it supplies the trace and + # metrics SDK components with the correct static-link ordering, which + # naming components individually does not (the package under-declares + # inter-component dependencies). + target_link_libraries( + xrpl.libxrpl.telemetry + PUBLIC opentelemetry-cpp::opentelemetry-cpp + ) +endif() + add_module(xrpl tx) -target_link_libraries(xrpl.libxrpl.tx PUBLIC xrpl.libxrpl.ledger) +target_link_libraries( + xrpl.libxrpl.tx + PUBLIC xrpl.libxrpl.ledger xrpl.libxrpl.telemetry +) add_library(xrpl.libxrpl) set_target_properties(xrpl.libxrpl PROPERTIES OUTPUT_NAME xrpl) @@ -233,6 +274,7 @@ target_link_modules( resource server shamap + telemetry tx ) diff --git a/cmake/XrplDocs.cmake b/cmake/XrplDocs.cmake index 7b3e9b3b306..6f81dcfd5f1 100644 --- a/cmake/XrplDocs.cmake +++ b/cmake/XrplDocs.cmake @@ -27,8 +27,12 @@ file( src/*.cpp src/*.md Builds/*.md - *.md ) +# Add only top-level .md files (README, CONTRIBUTING, etc.) without +# recursing into dot-directories like .claude/ whose files are not +# valid Doxygen/CMake sources. +file(GLOB doxygen_top_md CONFIGURE_DEPENDS "*.md") +list(APPEND doxygen_input ${doxygen_top_md}) list(APPEND doxygen_input external/README.md) set(dependencies "${doxygen_input}" "${doxyfile}") diff --git a/conan.lock b/conan.lock index c6a4070c771..5b6fa9620fa 100644 --- a/conan.lock +++ b/conan.lock @@ -10,11 +10,14 @@ "rocksdb/10.5.1#4a197eca381a3e5ae8adf8cffa5aacd0%1782392413.075713", "re2/20251105#8579cfd0bda4daf0683f9e3898f964b4%1782392402.431897", "protobuf/6.33.5#ff253ead763bd8d9904a52979cd21e81%1782392410.233933", + "opentelemetry-cpp/1.26.0#9d81768342c78cb897345fd419b358d2%1776934712.672", "openssl/3.6.3#f806de8933e3bf6f01016c6a888cee2e%1783945160.863288", "nudb/2.0.9#11149c73f8f2baff9a0198fe25971fc7%1782392402.297166", + "nlohmann_json/3.11.3#45828be26eb619a2e04ca517bb7b828d%1701220705.259", "mpt-crypto/0.4.0-rc4#ffdba12f2332357f0d8b0ae944cfff52%1784138702.932355", "lz4/1.10.0#982d9b673900f665a1da109e09c17cab%1782392402.164188", "libiconv/1.17#9923bc6dc6f106646d6967e0039a5ada%1782392792.775744", + "libcurl/8.20.0#c90b0c91a33d9a79b519c1c70bafc823%1780907438.587", "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1782392402.420732", "libarchive/3.8.7#c446109bd1f1d8ba7936c94189bc50e6%1782392403.066892", "jemalloc/5.3.1#1fc58d55316041f10fbc1e8a2eae632a%1776700028.228", @@ -32,9 +35,15 @@ "zlib/1.3.2#1cb806da49011867778ffb6ac7190fcb%1782392402.122708", "strawberryperl/5.32.1.1#8d114504d172cfea8ea1662d09b6333e%1782395692.540639", "protobuf/6.33.5#ff253ead763bd8d9904a52979cd21e81%1782392410.233933", + "pkgconf/2.5.1#93c2051284cba1279494a43a4fcfeae2%1757684701.089", + "opentelemetry-proto/1.7.0#ed6d5bd761bef0afb0ba09676420b9ea%1749461220.268", + "ninja/1.13.2#c8c5dc2a52ed6e4e42a66d75b4717ceb%1764096931.974", "nasm/2.16.01#31e26f2ee3c4346ecd347911bd126904%1782395690.33162", "msys2/cci.latest#d22fe7b2808f5fd34d0a7923ace9c54f%1770657326.649", + "meson/1.10.2#9d2d10681fe7fe61c788c58626c89b25%1775558003.754", "m4/1.4.19#34c4bbc3eeebe98ca6edf2f52d602e7d%1777282960.259", + "libtool/2.4.7#14e7739cc128bc1623d2ed318008e47e%1755679003.847", + "gnu-config/cci.20210814#466e9d4d7779e1c142443f7ea44b4284%1762363589.329", "cmake/4.3.3#840cf00ea09777e05c2050a50a82c722%1782392418.696091", "b2/5.4.2#ffd6084a119587e70f11cd45d1a386e2%1782392402.624226", "automake/1.16.5#b91b7c384c3deaa9d535be02da14d04f%1755524470.56", @@ -60,6 +69,9 @@ ], "lz4/[>=1.9.4 <2]": [ "lz4/1.10.0#982d9b673900f665a1da109e09c17cab" + ], + "protobuf/[>=4.25.3 <7]": [ + "protobuf/6.33.5#ff253ead763bd8d9904a52979cd21e81" ] }, "config_requires": [] diff --git a/conan/profiles/default b/conan/profiles/default index ae6e23c3c3d..cc937ce5111 100644 --- a/conan/profiles/default +++ b/conan/profiles/default @@ -25,3 +25,18 @@ compiler.libcxx={{ detect_api.detect_libcxx(compiler, version, compiler_exe) }} {# More info: https://docs.conan.io/2/reference/extensions/binary_compatibility.html #} user.package:cppstd_version=23 tools.info.package_id:confs+=["user.package:cppstd_version"] +{% if compiler == "gcc" and compiler_version < 13 %} +tools.build:cxxflags+=['-Wno-restrict'] +{% endif %} +{% if os == "Windows" %} +# opentelemetry-cpp's recipe removes the `shared` option on Windows and never +# sets BUILD_SHARED_LIBS, so its upstream CMake defaults the protobuf-generated +# `opentelemetry_proto` target to a DLL (opentelemetry_proto.dll). The rest of +# the project links statically and nothing deploys that DLL next to the +# executables, so the telemetry unit test fails to start with +# STATUS_DLL_NOT_FOUND (0xC0000135). Force the dependency to build fully static +# so no runtime DLL is produced. The conf is folded into the package id so a +# fresh static binary is built instead of reusing a previously cached one. +opentelemetry-cpp/*:tools.cmake.cmaketoolchain:extra_variables={"BUILD_SHARED_LIBS": "OFF"} +opentelemetry-cpp/*:tools.info.package_id:confs+=["tools.cmake.cmaketoolchain:extra_variables"] +{% endif %} diff --git a/conanfile.py b/conanfile.py index f883761f0e3..fdd74264408 100644 --- a/conanfile.py +++ b/conanfile.py @@ -22,6 +22,7 @@ class Xrpl(ConanFile): "rocksdb": [True, False], "shared": [True, False], "static": [True, False], + "telemetry": [True, False], "tests": [True, False], "unity": [True, False], "xrpld": [True, False], @@ -54,6 +55,7 @@ class Xrpl(ConanFile): "rocksdb": True, "shared": False, "static": True, + "telemetry": True, "tests": False, "unity": False, "xrpld": False, @@ -144,6 +146,10 @@ def requirements(self): self.requires("rocksdb/10.5.1") self.requires("secp256k1/0.7.1", transitive_headers=True) self.requires("sqlite3/3.53.0", force=True) + # OpenTelemetry C++ SDK for distributed tracing (optional). + # Provides OTLP/HTTP exporter, batch span processor, and trace API. + if self.options.telemetry: + self.requires("opentelemetry-cpp/1.26.0") self.requires("xxhash/0.8.3", transitive_headers=True) exports_sources = ( @@ -173,6 +179,7 @@ def generate(self): tc.variables["rocksdb"] = self.options.rocksdb tc.variables["BUILD_SHARED_LIBS"] = self.options.shared tc.variables["static"] = self.options.static + tc.variables["telemetry"] = self.options.telemetry tc.variables["unity"] = self.options.unity tc.variables["xrpld"] = self.options.xrpld tc.generate() @@ -226,3 +233,5 @@ def package_info(self): ] if self.options.rocksdb: libxrpl.requires.append("rocksdb::librocksdb") + if self.options.telemetry: + libxrpl.requires.append("opentelemetry-cpp::opentelemetry-cpp") diff --git a/docker/telemetry/.env.alerting.example b/docker/telemetry/.env.alerting.example new file mode 100644 index 00000000000..9822c635f1d --- /dev/null +++ b/docker/telemetry/.env.alerting.example @@ -0,0 +1,32 @@ +# rippled OTel alerting delivery — copy to `.env.alerting` and fill in. +# +# `.env.alerting` is gitignored; never commit a real Slack webhook or address. +# Grafana reads these when the stack starts and expands the ${VARS} referenced +# in grafana/provisioning/alerting/contactpoints.yaml. See the Alerting +# section of docs/telemetry-runbook.md. +# +# Any var left blank simply disables that delivery path — the stack still runs. + +# --- Slack --- +# Incoming-webhook URL from the Slack app (Incoming Webhooks feature). +# Used by both the warning (xrpld-default) and critical (xrpld-critical) tiers. +SLACK_WEBHOOK_URL= + +# Channel label. With an incoming webhook the target channel is fixed by the +# webhook itself; this only satisfies Grafana's Slack validator. Defaults to +# #xrpld-alerts if unset. +SLACK_CHANNEL=#xrpld-alerts + +# --- Email (critical tier only) --- +# Comma-separated recipient list for critical alerts. +ALERT_EMAIL_TO= + +# SMTP relay Grafana sends through. Email only delivers when SMTP is enabled +# and these point at a real relay (e.g. an internal smart host or an +# authenticated provider). +GF_SMTP_ENABLED=false +GF_SMTP_HOST=smtp.example.com:587 +GF_SMTP_USER= +GF_SMTP_PASSWORD= +GF_SMTP_FROM_ADDRESS=alerts@xrpld.local +GF_SMTP_FROM_NAME=xrpld Alerts diff --git a/docker/telemetry/.env.grafanacloud-alloy.example b/docker/telemetry/.env.grafanacloud-alloy.example new file mode 100644 index 00000000000..a43e09e6d92 --- /dev/null +++ b/docker/telemetry/.env.grafanacloud-alloy.example @@ -0,0 +1,27 @@ +# Grafana Alloy -> Grafana Cloud credentials for docker/telemetry/alloy/config.alloy. +# Copy to `.env.grafanacloud-alloy`, fill in, and source before starting Alloy. +# `.env.grafanacloud-alloy` is gitignored; never commit real tokens or stack ids. +# +# Prometheus values: Grafana Cloud -> Connections -> "Prometheus". +# OTLP values: Grafana Cloud -> Connections -> "OpenTelemetry (OTLP)". + +# --- Host / systemd metrics (prometheus.remote_write) --- +# remote_write push URL, e.g. https://prometheus-prod-XX-.grafana.net/api/prom/push +GRAFANACLOUD_PROM_URL= +# Numeric instance/stack id (Basic-auth username). +GRAFANACLOUD_PROM_USER= +# Access-policy token with metrics:write (Basic-auth password). +GRAFANACLOUD_PROM_KEY= + +# --- xrpld OTLP (otelcol.exporter.otlphttp) --- +# OTLP/HTTP gateway URL including the /otlp path, +# e.g. https://otlp-gateway-prod-.grafana.net/otlp +GRAFANACLOUD_OTLP_URL= +# Numeric instance/stack id (Basic-auth username). +GRAFANACLOUD_OTLP_USER= +# Access-policy token with metrics:write + traces:write (Basic-auth password). +GRAFANACLOUD_OTLP_KEY= + +# --- Per-node label --- +# host label applied to this node's scraped metrics (e.g. the node's hostname). +XRPLD_HOST_LABEL= diff --git a/docker/telemetry/.env.grafanacloud.example b/docker/telemetry/.env.grafanacloud.example new file mode 100644 index 00000000000..53a1a2a7f39 --- /dev/null +++ b/docker/telemetry/.env.grafanacloud.example @@ -0,0 +1,16 @@ +# Grafana Cloud OTLP credentials — copy to `.env.grafanacloud` and fill in. +# Find all three under Grafana Cloud -> Connections -> "OpenTelemetry (OTLP)". +# +# `.env.grafanacloud` is gitignored; never commit real tokens. + +# OTLP/HTTP gateway URL for your stack, including the /otlp path. +# Example: https://otlp-gateway-prod-us-east-0.grafana.net/otlp +GRAFANA_CLOUD_OTLP_ENDPOINT= + +# Numeric instance/stack id shown on the OTLP connection page +# (used as the Basic-auth username). +GRAFANA_CLOUD_INSTANCE_ID= + +# A Cloud Access Policy token with metrics:write, traces:write, logs:write +# (used as the Basic-auth password). +GRAFANA_CLOUD_API_TOKEN= diff --git a/docker/telemetry/.gitignore b/docker/telemetry/.gitignore new file mode 100644 index 00000000000..41139c450af --- /dev/null +++ b/docker/telemetry/.gitignore @@ -0,0 +1,13 @@ +# Runtime data generated by xrpld and telemetry stack +data/ + +# Env. file carrying environmental setup data for local or cloud runs. +.env.* + +# Keep examples +!.env.alerting.example +!.env.grafanacloud.example +!.env.grafanacloud-alloy.example + +# Do not commit grafana cloud versions with default filters +grafanacloud/*.json diff --git a/docker/telemetry/TESTING.md b/docker/telemetry/TESTING.md new file mode 100644 index 00000000000..7cf2b19fa0d --- /dev/null +++ b/docker/telemetry/TESTING.md @@ -0,0 +1,708 @@ +# OpenTelemetry Integration Testing Guide + +This document describes how to verify the xrpld OpenTelemetry telemetry +pipeline end-to-end, from span generation through the observability stack +(otel-collector, Tempo, Prometheus, Grafana). + +--- + +## Prerequisites + +### Build xrpld with telemetry + +```bash +conan install . --build=missing -o telemetry=True +cmake --preset default -Dtelemetry=ON +cmake --build --preset default --target xrpld +``` + +The binary is at `.build/xrpld`. + +### Required tools + +- **Docker** with `docker compose` (v2) +- **curl** +- **jq** (JSON processor) + +### Verify binary + +```bash +.build/xrpld --version +``` + +--- + +## Test 1: Single-Node Standalone (Quick Verification) + +This test verifies RPC and transaction spans in standalone mode. Consensus +spans will not fire because standalone mode does not run consensus. + +### Step 1: Start the observability stack + +```bash +docker compose -f docker/telemetry/docker-compose.yml up -d +``` + +Wait for services to be ready: + +```bash +# otel-collector health +curl -sf http://localhost:13133/ && echo "collector ready" + +# Tempo readiness +curl -sf http://localhost:3200/ready >/dev/null && echo "tempo ready" +``` + +### Step 2: Start xrpld in standalone mode + +```bash +.build/xrpld --conf docker/telemetry/xrpld-telemetry.cfg -a --start +``` + +Wait a few seconds for the node to initialize. + +### Step 3: Exercise RPC spans + +```bash +# server_info +curl -s http://localhost:5005 \ + -d '{"method":"server_info"}' | jq .result.info.server_state + +# server_state +curl -s http://localhost:5005 \ + -d '{"method":"server_state"}' | jq .result.state.server_state + +# ledger +curl -s http://localhost:5005 \ + -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' | + jq .result.ledger_current_index +``` + +### Step 4: Submit a transaction + +Close the ledger first (required in standalone mode): + +```bash +curl -s http://localhost:5005 -d '{"method":"ledger_accept"}' +``` + +Submit a Payment from the genesis account: + +```bash +curl -s http://localhost:5005 -d '{ + "method": "submit", + "params": [{ + "secret": "snoPBrXtMeMyMHUVTgbuqAfg1SUTb", + "tx_json": { + "TransactionType": "Payment", + "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", + "Destination": "rPMh7Pi9ct699iZUTWzJaUMR1o42VEfGqF", + "Amount": "10000000" + } + }] +}' | jq .result.engine_result +``` + +Expected result: `"tesSUCCESS"`. + +Close the ledger again to finalize: + +```bash +curl -s http://localhost:5005 -d '{"method":"ledger_accept"}' +``` + +### Step 5: Verify traces in Tempo + +Wait 5 seconds for the batch export, then: + +```bash +TEMPO="http://localhost:3200" + +# Check xrpld service is registered +curl -s "$TEMPO/api/v2/search/tag/resource.service.name/values" | jq '.tagValues[].value' + +# Check RPC spans +curl -s "$TEMPO/api/search" \ + --data-urlencode 'q={resource.service.name="xrpld" && name="rpc.http_request"}' \ + --data-urlencode 'limit=5' | jq '.traces | length' + +curl -s "$TEMPO/api/search" \ + --data-urlencode 'q={resource.service.name="xrpld" && name="rpc.process"}' \ + --data-urlencode 'limit=5' | jq '.traces | length' + +curl -s "$TEMPO/api/search" \ + --data-urlencode 'q={resource.service.name="xrpld" && name="rpc.command.server_info"}' \ + --data-urlencode 'limit=5' | jq '.traces | length' + +# Check transaction spans +curl -s "$TEMPO/api/search" \ + --data-urlencode 'q={resource.service.name="xrpld" && name="tx.process"}' \ + --data-urlencode 'limit=5' | jq '.traces | length' +``` + +Or open Grafana Explore with Tempo datasource: http://localhost:3000 + +### Step 6: Teardown + +```bash +# Kill xrpld (Ctrl+C or) +kill $(pgrep -f 'xrpld.*xrpld-telemetry') + +# Stop observability stack +docker compose -f docker/telemetry/docker-compose.yml down + +# Clean xrpld data +rm -rf data/ +``` + +### Expected spans (standalone mode) + +| Span Name | Expected | Notes | +| --------------------------- | -------- | ----------------------------- | +| `rpc.http_request` | Yes | Every HTTP RPC call | +| `rpc.process` | Yes | Every RPC processing | +| `rpc.command.server_info` | Yes | server_info RPC | +| `rpc.command.server_state` | Yes | server_state RPC | +| `rpc.command.ledger` | Yes | ledger RPC | +| `rpc.command.submit` | Yes | submit RPC | +| `rpc.command.ledger_accept` | Yes | ledger_accept RPC | +| `tx.process` | Yes | Transaction submission | +| `tx.receive` | No | No peers in standalone | +| `consensus.*` | No | Consensus disabled standalone | + +--- + +## Test 2: 6-Node Consensus Network (Full Verification) + +This test verifies ALL span categories including consensus and peer +transaction relay, using a 6-node validator network. + +### Automated + +Run the integration test script: + +```bash +bash docker/telemetry/integration-test.sh +``` + +The script will: + +1. Start the observability stack +2. Generate 6 validator key pairs +3. Create config files for each node +4. Start all 6 nodes +5. Wait for consensus ("proposing" state) +6. Exercise RPC, submit transactions +7. Verify all span categories in Tempo +8. Verify spanmetrics in Prometheus +9. Print results and leave the stack running + +### Manual + +If you prefer to run the steps manually: + +#### Step 1: Start observability stack + +```bash +docker compose -f docker/telemetry/docker-compose.yml up -d +``` + +#### Step 2: Generate validator keys + +Start a temporary standalone xrpld: + +```bash +.build/xrpld --conf docker/telemetry/xrpld-telemetry.cfg -a --start & +TEMP_PID=$! +sleep 5 +``` + +Generate 6 key pairs: + +```bash +for i in $(seq 1 6); do + curl -s http://localhost:5005 \ + -d '{"method":"validation_create"}' | jq '.result' +done +``` + +Record the `validation_seed` and `validation_public_key` for each. +Kill the temporary node: + +```bash +kill $TEMP_PID +rm -rf data/ +``` + +#### Step 3: Create node configs + +For each node (1-6), create a config file. Template: + +```ini +[server] +port_rpc +port_peer + +[port_rpc] +port = {5004 + node_number} +ip = 127.0.0.1 +admin = 127.0.0.1 +protocol = http + +[port_peer] +port = {51234 + node_number} +ip = 0.0.0.0 +protocol = peer + +[node_db] +type=NuDB +path=/tmp/xrpld-integration/node{N}/nudb +online_delete=256 + +[database_path] +/tmp/xrpld-integration/node{N}/db + +[debug_logfile] +/tmp/xrpld-integration/node{N}/debug.log + +[validation_seed] +{seed from step 2} + +[validators_file] +/tmp/xrpld-integration/validators.txt + +[ips_fixed] +127.0.0.1 51235 +127.0.0.1 51236 +127.0.0.1 51237 +127.0.0.1 51238 +127.0.0.1 51239 +127.0.0.1 51240 + +[peer_private] +1 + +[telemetry] +enabled=1 +endpoint=http://localhost:4318/v1/traces +batch_size=512 +batch_delay_ms=2000 +max_queue_size=2048 +trace_rpc=1 +trace_transactions=1 +trace_consensus=1 +trace_peer=1 +trace_ledger=1 + +[rpc_startup] +{ "command": "log_level", "severity": "warning" } + +[ssl_verify] +0 +``` + +#### Step 4: Create validators.txt + +```ini +[validators] +{public_key_1} +{public_key_2} +{public_key_3} +{public_key_4} +{public_key_5} +{public_key_6} +``` + +#### Step 5: Start all 6 nodes + +```bash +for i in $(seq 1 6); do + .build/xrpld --conf /tmp/xrpld-integration/node$i/xrpld.cfg --start & + echo $! >/tmp/xrpld-integration/node$i/xrpld.pid +done +``` + +#### Step 6: Wait for consensus + +Poll each node until `server_state` = `"proposing"`: + +```bash +for port in 5005 5006 5007 5008 5009 5010; do + while true; do + state=$(curl -s http://localhost:$port \ + -d '{"method":"server_info"}' | + jq -r '.result.info.server_state') + echo "Port $port: $state" + [ "$state" = "proposing" ] && break + sleep 5 + done +done +``` + +#### Step 7: Exercise RPC and submit transaction + +```bash +# RPC calls +curl -s http://localhost:5005 -d '{"method":"server_info"}' +curl -s http://localhost:5005 -d '{"method":"server_state"}' +curl -s http://localhost:5005 -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' + +# Submit transaction +curl -s http://localhost:5005 -d '{ + "method": "submit", + "params": [{ + "secret": "snoPBrXtMeMyMHUVTgbuqAfg1SUTb", + "tx_json": { + "TransactionType": "Payment", + "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", + "Destination": "rPMh7Pi9ct699iZUTWzJaUMR1o42VEfGqF", + "Amount": "10000000" + } + }] +}' +``` + +Wait 15 seconds for consensus and batch export. + +#### Step 8: Verify in Tempo + +See the "Verification Queries" section below. + +--- + +## Expected Span Catalog + +All 16 production span names instrumented across Phases 2-5: + +| Span Name | Source File | Phase | Key Attributes | How to Trigger | +| --------------------------- | ----------------- | ----- | ---------------------------------------------------------------------------------------- | ------------------------- | +| `rpc.http_request` | ServerHandler.cpp | 2 | -- | Any HTTP RPC call | +| `rpc.ws_upgrade` | ServerHandler.cpp | 2 | -- | WebSocket upgrade | +| `rpc.ws_message` | ServerHandler.cpp | 2 | -- | WebSocket RPC message | +| `rpc.process` | ServerHandler.cpp | 2 | -- | RPC processing | +| `rpc.command.` | RPCHandler.cpp | 2 | `command`, `version`, `rpc_role` | Any RPC command | +| `tx.process` | NetworkOPs.cpp | 3 | `xrpl.tx.hash`, `local`, `path` | Submit transaction | +| `tx.receive` | PeerImp.cpp | 3 | `xrpl.peer.id` | Peer relays transaction | +| `consensus.proposal.send` | RCLConsensus.cpp | 4 | `xrpl.consensus.round` | Consensus proposing phase | +| `consensus.ledger_close` | RCLConsensus.cpp | 4 | `xrpl.consensus.ledger.seq`, `xrpl.consensus.mode` | Ledger close event | +| `consensus.accept` | RCLConsensus.cpp | 4 | `xrpl.consensus.proposers`, `xrpl.consensus.round_time_ms` | Ledger accepted | +| `consensus.validation.send` | RCLConsensus.cpp | 4 | `xrpl.consensus.ledger.seq`, `xrpl.consensus.proposing` | Validation sent | +| `consensus.accept.apply` | RCLConsensus.cpp | 4 | `xrpl.consensus.close_time`, `close_time_correct`, `close_resolution_ms`, `state` | Ledger apply + close time | +| `tx.apply` | BuildLedger.cpp | 5 | `xrpl.ledger.tx_count`, `xrpl.ledger.tx_failed` | Ledger close (tx set) | +| `ledger.build` | BuildLedger.cpp | 5 | `xrpl.ledger.seq`, `xrpl.ledger.close_time`, `close_time_correct`, `close_resolution_ms` | Ledger build | +| `ledger.validate` | LedgerMaster.cpp | 5 | `xrpl.ledger.seq`, `xrpl.ledger.validations` | Ledger validated | +| `ledger.store` | LedgerMaster.cpp | 5 | `xrpl.ledger.seq` | Ledger stored | +| `peer.proposal.receive` | PeerImp.cpp | 5 | `xrpl.peer.id`, `proposal_trusted` | Peer sends proposal | +| `peer.validation.receive` | PeerImp.cpp | 5 | `xrpl.peer.id`, `validation_trusted` | Peer sends validation | + +--- + +## Verification Queries + +### Tempo API + +Base URL: `http://localhost:3200` + +```bash +TEMPO="http://localhost:3200" + +# List all services +curl -s "$TEMPO/api/v2/search/tag/resource.service.name/values" | jq '.tagValues[].value' + +# Query traces by operation +for op in "rpc.http_request" "rpc.ws_upgrade" "rpc.ws_message" "rpc.process" \ + "rpc.command.server_info" "rpc.command.server_state" "rpc.command.ledger" \ + "tx.process" "tx.receive" "tx.apply" \ + "consensus.proposal.send" "consensus.ledger_close" \ + "consensus.accept" "consensus.accept.apply" \ + "consensus.validation.send" \ + "ledger.build" "ledger.validate" "ledger.store" \ + "peer.proposal.receive" "peer.validation.receive"; do + count=$(curl -s "$TEMPO/api/search" \ + --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \ + --data-urlencode "limit=5" | + jq '.traces | length') + printf "%-35s %s traces\n" "$op" "$count" +done +``` + +### Prometheus API + +Base URL: `http://localhost:9090` + +```bash +PROM="http://localhost:9090" + +# Span call counts (from spanmetrics connector) +curl -s "$PROM/api/v1/query?query=span_calls_total" | + jq '.data.result[] | {span: .metric.span_name, count: .value[1]}' + +# Latency histogram +curl -s "$PROM/api/v1/query?query=span_duration_milliseconds_count" | + jq '.data.result[] | {span: .metric.span_name, count: .value[1]}' + +# RPC calls by command +curl -s "$PROM/api/v1/query?query=span_calls_total{span_name=~\"rpc.command.*\"}" | + jq '.data.result[] | {command: .metric["command"], count: .value[1]}' + +# Deployment-tier labels present on metrics (set by the collector's +# resource/tier processor and promoted via resource_to_telemetry_conversion). +# Expect deployment_environment and xrpl_network_type on each series. +curl -s "$PROM/api/v1/query?query=span_calls_total" | + jq '.data.result[0].metric | {deployment_environment, xrpl_network_type, service_name}' +``` + +### Grafana + +Open http://localhost:3000 (anonymous admin access enabled). + +Pre-configured dashboards: + +- **RPC Performance**: Request rates, latency percentiles by command, top commands, WebSocket rate +- **Transaction Overview**: Transaction processing rates, apply duration, peer relay, failed tx rate +- **Consensus Health**: Consensus round duration, proposer counts, mode tracking, accept heatmap +- **Ledger Operations**: Build/validate/store rates and durations, TX apply metrics +- **Peer Network**: Proposal/validation receive rates, trusted vs untrusted breakdown (requires `trace_peer=1`) + +Pre-configured datasources: + +- **Tempo**: Trace data at `http://tempo:3200` +- **Prometheus**: Metrics at `http://prometheus:9090` +- **Loki**: Log data at `http://loki:3100` (via Grafana Explore) + +--- + +## Exporting to Grafana Cloud + +Instead of (or alongside) the local backends, the collector can forward +traces, metrics, and logs to a hosted **Grafana Cloud** stack. This is a +runtime choice layered on top of the base stack — xrpld and the base +`docker-compose.yml` are unchanged. + +### Step 1: Get Grafana Cloud OTLP credentials + +From **Grafana Cloud → Connections → OpenTelemetry (OTLP)**, note the OTLP +gateway endpoint (ends in `/otlp`), the numeric instance id, and an +access-policy token with `metrics:write`, `traces:write`, and `logs:write`. + +### Step 2: Fill in the env file + +```bash +cp docker/telemetry/.env.grafanacloud.example docker/telemetry/.env.grafanacloud +# edit .env.grafanacloud: +# GRAFANA_CLOUD_OTLP_ENDPOINT=https://otlp-gateway-.grafana.net/otlp +# GRAFANA_CLOUD_INSTANCE_ID= +# GRAFANA_CLOUD_API_TOKEN= +``` + +`.env.grafanacloud` is gitignored — never commit real tokens. + +### Step 3: Start the stack with cloud export enabled + +```bash +docker compose -f docker/telemetry/docker-compose.yml \ + -f docker/telemetry/docker-compose.grafanacloud.yaml up -d +``` + +The override swaps the collector onto `otel-collector-config.grafanacloud.yaml`, +which keeps the local Tempo/Prometheus/Loki exporters and adds one +OTLP/HTTP exporter to Grafana Cloud on all three pipelines. Bring the stack +up with just the base file to return to local-only. + +### Step 4: Verify data reaches Grafana Cloud + +After exercising RPC/transaction workflows (Tests 1 or 2), open your Grafana +Cloud instance and confirm: + +- **Traces**: Explore → hosted Tempo datasource → search `{resource.service.name="xrpld"}` +- **Metrics**: Explore → hosted Prometheus/Mimir → query `span_calls_total` +- **Logs**: Explore → hosted Loki → query `{job="xrpld"}` (requires `warning`+ file logging) + +If nothing appears, check the collector logs for auth/export errors: + +```bash +docker compose -f docker/telemetry/docker-compose.yml \ + -f docker/telemetry/docker-compose.grafanacloud.yaml \ + logs otel-collector | grep -iE 'grafanacloud|401|403|export' +``` + +A `401`/`403` means the instance id or token is wrong; a connection error +means the endpoint URL is wrong or missing the `/otlp` path. + +--- + +## Test 3: Log-Trace Correlation (Phase 8) + +Phase 8 injects `trace_id` and `span_id` into xrpld's log output when +a log line is emitted within an active OTel span. This test verifies the +end-to-end log-trace correlation pipeline. + +### Step 1: Verify trace_id in log output + +After running Test 1 or Test 2 (which generate RPC spans), check the +xrpld debug.log for trace context: + +```bash +grep 'trace_id=[a-f0-9]\{32\} span_id=[a-f0-9]\{16\}' /path/to/debug.log +``` + +Expected: log lines with `trace_id=<32hex> span_id=<16hex>` between the +severity code and the message. Example: + +``` +2024-Jan-15 10:30:45.123456 UTC RPCHandler:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Calling server_info +``` + +Lines emitted outside of an active span (background tasks, startup) will +NOT have trace context — this is expected. + +### Step 2: Cross-check trace_id in Tempo + +Extract a `trace_id` from the log and verify it exists in Tempo: + +```bash +TRACE_ID=$(grep -o 'trace_id=[a-f0-9]\{32\}' /path/to/debug.log | head -1 | cut -d= -f2) +echo "Checking trace: $TRACE_ID" +curl -s "http://localhost:3200/api/traces/$TRACE_ID" | jq '.batches | length' +``` + +Expected result: `> 0` (the trace exists in Tempo). + +### Step 3: Verify Loki log ingestion + +The OTel Collector's filelog receiver tails xrpld's debug.log and +exports parsed entries to Loki. Verify Loki has received entries: + +```bash +# Query Loki for any xrpld logs +curl -sG "http://localhost:3100/loki/api/v1/query" \ + --data-urlencode 'query={job="xrpld"}' \ + --data-urlencode 'limit=5' | jq '.data.result | length' +``` + +Expected: > 0 results. + +### Step 4: Verify Grafana Tempo-to-Loki correlation + +1. Open Grafana at http://localhost:3000 +2. Navigate to **Explore** -> select **Tempo** datasource +3. Search for a trace (e.g., operation `rpc.command.server_info`) +4. Click **"Logs for this trace"** in the trace detail view +5. Verify that Loki log lines appear, filtered by the trace's `trace_id` + +### Step 5: Verify Grafana Loki-to-Tempo correlation + +1. In Grafana **Explore**, select **Loki** datasource +2. Query: `{job="xrpld"} |= "trace_id="` +3. In the log results, click the **TraceID** derived field link +4. Verify it navigates to the full trace in Tempo + +### Expected results + +| Check | Expected | +| ------------------------------ | ---------------------------------------- | +| `trace_id=` in debug.log | Present in log lines within active spans | +| `span_id=` in debug.log | Present alongside trace_id | +| Logs without active span | No trace_id/span_id fields | +| trace_id in Tempo | Matches a valid trace | +| Loki log ingestion | Logs visible via LogQL | +| Tempo -> Loki "Logs for trace" | Shows correlated log lines | +| Loki -> Tempo TraceID link | Navigates to correct trace | + +--- + +## Troubleshooting + +### No traces in Tempo + +1. Check otel-collector logs: + ```bash + docker compose -f docker/telemetry/docker-compose.yml logs otel-collector + ``` +2. Verify xrpld telemetry config has `enabled=1` and correct endpoint +3. Check that otel-collector port 4318 is accessible: + ```bash + curl -sf http://localhost:4318 && echo "reachable" + ``` +4. Increase `batch_delay_ms` or decrease `batch_size` in xrpld config + +### Nodes not reaching "proposing" state + +1. Check that all peer ports (51235-51240) are not in use: + ```bash + for p in 51235 51236 51237 51238 51239 51240; do + ss -tlnp | grep ":$p " && echo "port $p in use" + done + ``` +2. Verify `[ips_fixed]` lists all 6 peer ports +3. Verify `validators.txt` has all 6 public keys +4. Check node debug logs: `tail -50 /tmp/xrpld-integration/node1/debug.log` +5. Ensure `[peer_private]` is set to `1` (prevents reaching out to public network) + +### Transaction not processing + +1. Verify genesis account exists: + ```bash + curl -s http://localhost:5005 \ + -d '{"method":"account_info","params":[{"account":"rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"}]}' | + jq .result.account_data.Balance + ``` +2. Check submit response for error codes +3. In standalone mode, remember to call `ledger_accept` after submitting + +### No trace_id in log output (Phase 8) + +1. Verify xrpld was built with `telemetry=ON` (`-Dtelemetry=ON` in CMake) +2. Verify `enabled=1` in the `[telemetry]` config section +3. Log lines only contain trace context when emitted inside an active span. + Background logs (startup, periodic tasks outside spans) will not have + `trace_id`/`span_id`. +4. Ensure the trace category is enabled (e.g., `trace_rpc=1` for RPC logs) + +### No logs in Loki (Phase 8) + +1. Verify the log file mount in docker-compose.yml: + ```yaml + volumes: + - ${XRPLD_LOG_DIR:-./data/logs}:/var/log/xrpld:ro + ``` + The mount source defaults to the repo-relative `docker/telemetry/data/logs` + (where the telemetry configs write). Override `XRPLD_LOG_DIR` to tail logs + from another root. +2. Check OTel Collector logs for filelog receiver errors: + ```bash + docker compose -f docker/telemetry/docker-compose.yml logs otel-collector | grep -i "filelog\|loki\|error" + ``` +3. Verify Loki is running: + ```bash + curl -s http://localhost:3100/ready + ``` +4. Verify the filelog receiver glob pattern matches your log files: + The default pattern is `/var/log/xrpld/*/debug.log` + +### Grafana trace-log links not working (Phase 8) + +1. Verify `tracesToLogs` is configured in the Tempo datasource provisioning + (`docker/telemetry/grafana/provisioning/datasources/tempo.yaml`) +2. Verify `derivedFields` is configured in the Loki datasource provisioning + (`docker/telemetry/grafana/provisioning/datasources/loki.yaml`) +3. Restart Grafana after changing provisioning files: + ```bash + docker compose -f docker/telemetry/docker-compose.yml restart grafana + ``` + +### Spanmetrics not appearing in Prometheus + +1. Verify otel-collector config has `spanmetrics` connector +2. Check that the metrics pipeline is configured: + ```yaml + service: + pipelines: + metrics: + receivers: [spanmetrics] + exporters: [prometheus] + ``` +3. Verify Prometheus can reach collector: + ```bash + curl -s http://localhost:9090/api/v1/targets | jq '.data.activeTargets' + ``` diff --git a/docker/telemetry/alloy/config.alloy b/docker/telemetry/alloy/config.alloy new file mode 100644 index 00000000000..87cb52ca160 --- /dev/null +++ b/docker/telemetry/alloy/config.alloy @@ -0,0 +1,275 @@ +// Grafana Alloy collector config for an xrpld node. +// +// WHY THIS FILE EXISTS +// -------------------- +// Some deployments feed telemetry through Grafana Alloy instead of the +// reference OpenTelemetry Collector (otel-collector-config.grafanacloud.yaml). +// xrpld sends OTLP (traces + native beast::insight metrics) to Alloy, which +// forwards to the Grafana Cloud OTLP gateway. That path carries traces and +// native metrics, but on its own produces NO span-derived RED metrics +// (span_calls_total / span_duration_milliseconds_*): those are NOT emitted by +// xrpld -- they are derived by a spanmetrics connector from the spans. Without +// the connector a node is missing from every span-based Grafana dashboard. +// +// This config therefore mirrors the two stages the reference collector has: +// 1. resource tagging (service.name, deployment.environment, xrpl.network.type) +// 2. spanmetrics (derives the RED metrics the dashboards query) +// See docker/telemetry/otel-collector-config.grafanacloud.yaml for the +// authoritative collector equivalent; keep the dimension list in sync with it. +// +// PIPELINE +// +// HOST / SYSTEMD METRICS: +// unix exporter + custom scrape --> relabel(host) --> prometheus.remote_write +// +// xrpld OTLP: +// receiver.otlp ─▶ processor.transform.tier ─┬─▶ connector.spanmetrics ─┐ +// │ │ +// └─▶ processor.batch ◀───────┘ +// │ traces + metrics +// ▼ +// exporter.otlphttp (GC OTLP gateway) +// +// The Grafana Cloud OTLP gateway converts OTLP resource attributes to +// Prometheus labels server-side, so no otelcol.exporter.prometheus is needed. +// +// CONFIGURATION -- NO SECRETS OR TENANT IDS ARE HARD-CODED IN THIS FILE. +// All credentials, endpoints, usernames and the per-node host label are read +// from the environment (same pattern as .env.grafanacloud.example). Copy +// .env.grafanacloud-alloy.example to .env.grafanacloud-alloy, fill it in, and +// source it before starting Alloy. Required variables: +// GRAFANACLOUD_PROM_URL Prometheus remote_write push URL +// GRAFANACLOUD_PROM_USER remote_write basic-auth username (numeric stack id) +// GRAFANACLOUD_PROM_KEY remote_write basic-auth password (access token) +// GRAFANACLOUD_OTLP_URL OTLP/HTTP gateway URL, including the /otlp path +// GRAFANACLOUD_OTLP_USER OTLP basic-auth username (numeric stack id) +// GRAFANACLOUD_OTLP_KEY OTLP basic-auth password (access token) +// XRPLD_HOST_LABEL host label for this node's scraped metrics +// +// PER-DEPLOYMENT EDITS: the deployment.environment and xrpl.network.type tier +// values in otelcol.processor.transform are literals (OTTL cannot read env +// vars) -- edit them to match this node's tier and network. + +logging { + level = "info" +} + +// =========================================================================== +// HOST / SYSTEMD METRICS (Prometheus remote_write path) +// =========================================================================== + +prometheus.remote_write "grafanacloud" { + endpoint { + url = sys.env("GRAFANACLOUD_PROM_URL") + + basic_auth { + username = sys.env("GRAFANACLOUD_PROM_USER") + password = sys.env("GRAFANACLOUD_PROM_KEY") + } + } +} + +prometheus.exporter.unix "default" { + enable_collectors = ["systemd", "textfile"] + systemd { + unit_include = "(rippled|xrpld).*" + } + textfile { + directory = "/usr/local/bin/" + } +} + +prometheus.scrape "system_metrics" { + targets = prometheus.exporter.unix.default.targets + scrape_interval = "15s" + forward_to = [prometheus.relabel.hostname.receiver] +} + +prometheus.scrape "custom_rippled_metrics" { + targets = [{ + __address__ = "localhost:9256", + }] + scrape_interval = "15s" + forward_to = [prometheus.relabel.hostname.receiver] +} + +prometheus.relabel "hostname" { + forward_to = [prometheus.remote_write.grafanacloud.receiver] + + rule { + action = "replace" + source_labels = ["instance"] + replacement = sys.env("XRPLD_HOST_LABEL") + target_label = "host" + } +} + +// =========================================================================== +// xrpld OTLP -> Grafana Cloud (traces + native metrics + derived spanmetrics) +// =========================================================================== + +// Receiver: xrpld sends OTLP/HTTP on 4318 and gRPC on 4317 (localhost only). +otelcol.receiver.otlp "xrpld" { + grpc { + endpoint = "127.0.0.1:4317" + } + http { + endpoint = "127.0.0.1:4318" + } + + output { + // Both signals pass through resource tagging first so they leave with + // identical resource identity. + traces = [otelcol.processor.transform.tier.input] + metrics = [otelcol.processor.transform.tier.input] + } +} + +// Resource tagging (reference: resource/tier + resource/stripsdk). +// * service.name -> "xrpld". Also corrects the service_name="true" +// bool-binding bug on stale binaries: even an un-rebuilt node exports a +// correct service.name once it passes through here. +// * deployment.environment -> set from XRPLD_DEPLOYMENT_ENV (the collector +// IS the environment, so it is authoritative -> upsert). +// * xrpl.network.type -> set only when absent (don't overwrite the node's +// own value). OTTL `where ... == nil` gives insert (not upsert) semantics. +// * telemetry.sdk.* -> deleted (SDK noise). +otelcol.processor.transform "tier" { + error_mode = "ignore" + + // NOTE: statements are OTTL (raw strings) -- River sys.env() does NOT expand + // inside them, and OTTL has no env() converter. The tier values below are + // plain deployment labels (not secrets), so they are literals here. + // PER-DEPLOYMENT EDIT: set deployment.environment and xrpl.network.type to + // match this node's tier and network. + trace_statements { + context = "resource" + statements = [ + `set(attributes["service.name"], "xrpld")`, + `set(attributes["deployment.environment"], "prod")`, + `set(attributes["xrpl.network.type"], "mainnet") where attributes["xrpl.network.type"] == nil`, + `delete_key(attributes, "telemetry.sdk.language")`, + `delete_key(attributes, "telemetry.sdk.name")`, + `delete_key(attributes, "telemetry.sdk.version")`, + ] + } + + metric_statements { + context = "resource" + statements = [ + `set(attributes["service.name"], "xrpld")`, + `set(attributes["deployment.environment"], "prod")`, + `set(attributes["xrpl.network.type"], "mainnet") where attributes["xrpl.network.type"] == nil`, + `delete_key(attributes, "telemetry.sdk.language")`, + `delete_key(attributes, "telemetry.sdk.name")`, + `delete_key(attributes, "telemetry.sdk.version")`, + ] + } + + output { + // Traces fan out: to the batch/gateway path AND into the spanmetrics + // connector so the RED metrics are derived from the same tagged spans. + traces = [ + otelcol.processor.batch.xrpld.input, + otelcol.connector.spanmetrics.xrpld.input, + ] + // Native metrics go straight to the batch/gateway path. + metrics = [otelcol.processor.batch.xrpld.input] + } +} + +// Spanmetrics connector (reference: connectors.spanmetrics, namespace "span"). +// Derives span_calls_total and span_duration_milliseconds_* that every span +// dashboard queries. The dimension list and buckets are copied from the +// reference otel-collector-config.grafanacloud.yaml -- keep them in sync (a +// panel that groups by a dimension missing here renders empty). +otelcol.connector.spanmetrics "xrpld" { + namespace = "span" + + histogram { + unit = "ms" + explicit { + // Buckets MUST stay strictly ascending (the connector binary-searches + // them and silently misbuckets otherwise) and MUST match the two + // otel-collector-config*.yaml lists. Sub-second boundaries cover + // RPC/tx/ledger spans; 2s-4s resolve second-scale consensus spans + // (consensus.round ~3.9s, consensus.establish ~1.9s) that used to pile + // into one 1s-5s bucket; 10s/30s give the ledger.acquire catch-up tail + // (~17% exceeds 5s) a measurable home so its p95/p99 stop reading +Inf. + buckets = ["1ms", "5ms", "10ms", "25ms", "50ms", "100ms", "250ms", "500ms", "1s", "2s", "3s", "4s", "5s", "10s", "30s"] + } + } + + // RPC / transaction dimensions. + dimension { name = "command" } + dimension { name = "rpc_status" } + dimension { name = "tx_type" } + dimension { name = "ter_result" } + dimension { name = "stage" } + dimension { name = "txq_status" } + dimension { name = "load_type" } + dimension { name = "is_batch" } + + // Consensus dimensions. + dimension { name = "consensus_mode" } + dimension { name = "close_time_correct" } + dimension { name = "consensus_state" } + dimension { name = "mode_new" } + dimension { name = "consensus_stalled" } + dimension { name = "consensus_phase" } + dimension { name = "consensus_result" } + + // Overlay / peer dimensions. + dimension { name = "local" } + dimension { name = "suppressed" } + dimension { name = "proposal_trusted" } + dimension { name = "validation_trusted" } + + // gRPC surface dimensions. + dimension { name = "method" } + dimension { name = "grpc_role" } + dimension { name = "grpc_status" } + + // ledger.acquire dimensions. + dimension { name = "outcome" } + dimension { name = "acquire_reason" } + + output { + // Derived span metrics rejoin the metric stream at the batch processor. + metrics = [otelcol.processor.batch.xrpld.input] + } +} + +// Batch traces + metrics (native and span-derived) before export. +otelcol.processor.batch "xrpld" { + timeout = "1s" + send_batch_size = 1024 + + output { + traces = [otelcol.exporter.otlphttp.grafanacloud.input] + metrics = [otelcol.exporter.otlphttp.grafanacloud.input] + } +} + +// Grafana Cloud OTLP gateway auth + exporter. The gateway converts OTLP +// resource attributes to Prometheus labels server-side. +otelcol.auth.basic "grafanacloud" { + username = sys.env("GRAFANACLOUD_OTLP_USER") + password = sys.env("GRAFANACLOUD_OTLP_KEY") +} + +otelcol.exporter.otlphttp "grafanacloud" { + client { + endpoint = sys.env("GRAFANACLOUD_OTLP_URL") + auth = otelcol.auth.basic.grafanacloud.handler + } + retry_on_failure { + enabled = true + max_elapsed_time = "5m" + } + sending_queue { + enabled = true + num_consumers = 4 + queue_size = 1000 + } +} diff --git a/docker/telemetry/docker-compose.grafanacloud.yaml b/docker/telemetry/docker-compose.grafanacloud.yaml new file mode 100644 index 00000000000..b78f188d53d --- /dev/null +++ b/docker/telemetry/docker-compose.grafanacloud.yaml @@ -0,0 +1,28 @@ +# Compose override — enable Grafana Cloud export at runtime. +# +# Layer this on top of the base stack to swap the collector onto the +# Grafana Cloud dual-export config and inject the OTLP credentials. The +# base docker-compose.yml is unchanged, so plain `up` stays local-only. +# +# Usage: +# 1. Put your three Grafana Cloud values in docker/telemetry/.env.grafanacloud +# (template: .env.grafanacloud.example). That file is gitignored. +# 2. Bring the stack up with BOTH compose files: +# docker compose -f docker/telemetry/docker-compose.yml \ +# -f docker/telemetry/docker-compose.grafanacloud.yaml up -d +# 3. To go back to local-only, bring the stack up with just the base file. +# +# Log mount: inherited from the base compose (./data/logs -> /var/log/xrpld). +# Both nodes' xrpld cfgs write to docker/telemetry/data/logs//debug.log, +# so no per-box remap is needed here. + +services: + otel-collector: + volumes: + # Mount the Grafana Cloud collector config over the default path + # (replaces local-only config for this run; file on disk untouched). + - ./otel-collector-config.grafanacloud.yaml:/etc/otel-collector-config.yaml:ro + # Secrets from the env file, injected as container env and + # resolved by the collector via ${env:...}. + env_file: + - .env.grafanacloud diff --git a/docker/telemetry/docker-compose.workload.yaml b/docker/telemetry/docker-compose.workload.yaml new file mode 100644 index 00000000000..818c65f4c58 --- /dev/null +++ b/docker/telemetry/docker-compose.workload.yaml @@ -0,0 +1,103 @@ +# Docker Compose workload harness for Phase 10 telemetry validation. +# +# Runs a 5-node validator cluster with full OTel telemetry stack: +# - 5 rippled validator nodes (consensus network) +# - OTel Collector (traces + native OTLP metrics) +# - Tempo (trace backend + search API) +# - Prometheus (metrics) +# - Loki (log aggregation for log-trace correlation) +# - Grafana (dashboards + trace/log exploration) +# +# Usage: +# # Start the harness (requires pre-built xrpld image or mount binary): +# docker compose -f docker/telemetry/docker-compose.workload.yaml up -d +# +# # Or use the orchestrator: +# docker/telemetry/workload/run-full-validation.sh +# +# Prerequisites: +# - xrpld binary built with -DXRPL_ENABLE_TELEMETRY=ON +# - Validator keys generated via generate-validator-keys.sh +# - Node configs generated by run-full-validation.sh +# +# Note: No Docker healthchecks are defined here. The orchestrator script +# (run-full-validation.sh) polls each service endpoint directly from the +# host, which avoids issues with missing curl/wget in container images. + +services: + # --------------------------------------------------------------------------- + # Telemetry Backend Stack + # --------------------------------------------------------------------------- + + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + command: ["--config=/etc/otel-collector-config.yaml"] + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP (traces + beast::insight metrics) + - "8889:8889" # Prometheus metrics endpoint + - "13133:13133" # Health check + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro + # Mount the validation workdir so the filelog receiver can tail node + # logs. run-full-validation.sh sets XRPLD_LOG_DIR to its workdir; the + # default matches that workdir so a bare `docker compose up` also works. + - ${XRPLD_LOG_DIR:-/tmp/xrpld-validation}:/var/log/xrpld:ro + depends_on: + - tempo + networks: + - workload-net + + tempo: + image: grafana/tempo:2.7.2 + command: ["-config.file=/etc/tempo.yaml"] + ports: + - "3200:3200" # Tempo HTTP API + volumes: + - ./tempo.yaml:/etc/tempo.yaml:ro + - tempo-data:/var/tempo + networks: + - workload-net + + prometheus: + image: prom/prometheus:latest + ports: + - "9090:9090" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + depends_on: + - otel-collector + networks: + - workload-net + + loki: + image: grafana/loki:3.4.2 + ports: + - "3100:3100" # Loki HTTP API + command: ["-config.file=/etc/loki/local-config.yaml"] + networks: + - workload-net + + grafana: + image: grafana/grafana:latest + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + ports: + - "3000:3000" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - tempo + - prometheus + - loki + networks: + - workload-net + +volumes: + tempo-data: + +networks: + workload-net: + driver: bridge diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml new file mode 100644 index 00000000000..cffe3d139ac --- /dev/null +++ b/docker/telemetry/docker-compose.yml @@ -0,0 +1,153 @@ +# Docker Compose stack for xrpld OpenTelemetry observability. +# +# Provides services for local development: +# - otel-collector: receives OTLP traces from xrpld, batches and +# forwards them to Tempo. Also tails xrpld log files +# via filelog receiver and exports to Loki. Listens on ports +# 4317 (gRPC) and 4318 (HTTP). +# - tempo: Grafana Tempo tracing backend, queryable via Grafana Explore +# on port 3000. Recommended for production (S3/GCS storage, TraceQL). +# - loki: Grafana Loki log aggregation backend for centralized log +# ingestion and log-trace correlation (Phase 8). +# - grafana: dashboards on port 3000, pre-configured with Tempo, +# Prometheus, and Loki datasources. +# +# Usage: +# docker compose -f docker/telemetry/docker-compose.yml up -d +# +# Configure xrpld to export traces by adding to xrpld.cfg: +# [telemetry] +# enabled=1 +# endpoint=http://localhost:4318/v1/traces + +services: + # OpenTelemetry Collector: receives spans from xrpld via OTLP protocol, + # batches them for efficiency, and forwards to Tempo for storage. + otel-collector: + image: otel/opentelemetry-collector-contrib:0.121.0 + command: ["--config=/etc/otel-collector-config.yaml"] + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP (traces + native OTel metrics) + - "8889:8889" # Prometheus metrics (spanmetrics + OTLP) + # StatsD UDP port removed — beast::insight now uses native OTLP. + # Uncomment if using server=statsd fallback: + # - "8125:8125/udp" + volumes: + # Mount collector pipeline config (receivers → processors → exporters) + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro + # Mount the xrpld log root for the filelog receiver. The telemetry + # configs write to docker/telemetry/data/logs//debug.log, so + # the default source is the repo-relative ./data/logs — user-owned and + # needing no root, so `docker compose up` works with no setup. Override + # XRPLD_LOG_DIR to point at another root (e.g. the integration test sets + # it to its own workdir). Mounted read-only so the collector only tails. + - ${XRPLD_LOG_DIR:-./data/logs}:/var/log/xrpld:ro + depends_on: + - tempo + - loki + networks: + - xrpld-telemetry + + # Grafana Tempo: distributed tracing backend that stores and indexes + # spans. Queryable via TraceQL in Grafana Explore. + tempo: + image: grafana/tempo:2.7.2 + command: ["-config.file=/etc/tempo.yaml"] + ports: + - "3200:3200" # Tempo HTTP API (health check, query) + volumes: + # Mount Tempo storage and ingestion config + - ./tempo.yaml:/etc/tempo.yaml:ro + # Persistent volume for trace data (WAL + blocks) + - tempo-data:/var/tempo + networks: + - xrpld-telemetry + + # Grafana Loki for centralized log ingestion and log-trace + # correlation. Loki 3.x supports native OTLP ingestion, so the OTel + # Collector exports via otlphttp to Loki's /otlp endpoint. + # Query logs via Grafana Explore -> Loki at http://localhost:3000. + loki: + image: grafana/loki:3.4.2 + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + volumes: + - loki-data:/loki + networks: + - xrpld-telemetry + + prometheus: + # Pinned to the v2.53 LTS line for reproducible, config-stable runs. + image: prom/prometheus:v2.53.3 + ports: + - "9090:9090" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + depends_on: + - otel-collector + networks: + - xrpld-telemetry + + # Grafana: visualization UI with Tempo pre-configured as a datasource. + # Anonymous admin access enabled for local development convenience. + grafana: + image: grafana/grafana:11.5.2 + # Alerting secrets/addresses (Slack webhook, alert email) come from this + # gitignored file; Grafana expands the ${VARS} referenced in the alerting + # provisioning YAML. Absent file = unset vars = receivers have no live + # destination, which is fine for a local stack. See .env.alerting.example. + env_file: + - path: .env.alerting + required: false + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true # No login required for local dev + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin # Full access without auth + # Remote image rendering: point Grafana at the renderer container. + # These belong on the grafana service (the server delegates renders); + # the callback URL is how the renderer fetches the panel from grafana. + - GF_RENDERING_SERVER_URL=http://renderer:8081/render + - GF_RENDERING_CALLBACK_URL=http://grafana:3000/ + # SMTP for the critical-tier email receiver and the Slack webhook / email + # address for the contact points all come from the env_file above, which + # injects them straight into the container environment for Grafana to + # expand. They are deliberately NOT duplicated here: a compose-level + # `environment:` entry is interpolated from the host shell and would + # override (and blank out) the env_file values. + ports: + - "3000:3000" # Grafana web UI + volumes: + # Auto-provision Tempo datasource and search filters on startup + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - tempo + - prometheus + - loki + - renderer + networks: + - xrpld-telemetry + + # Grafana image renderer: a sidecar that renders panels/dashboards to PNG + # for image export and alerting. Grafana calls it at http://renderer:8081. + renderer: + image: grafana/grafana-image-renderer:latest + ports: + - "8081:8081" # Renderer HTTP endpoint (called by grafana) + networks: + - xrpld-telemetry +# Named volume for Tempo trace storage (WAL and compacted blocks). +# Data persists across container restarts. Remove with: +# docker compose -f docker/telemetry/docker-compose.yml down -v +volumes: + tempo-data: + prometheus-data: + loki-data: + +# Isolated bridge network so services communicate by container name +# (e.g., the collector reaches Tempo at http://tempo:4317). +networks: + xrpld-telemetry: + driver: bridge diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json new file mode 100644 index 00000000000..06557080fd7 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -0,0 +1,1337 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Validation Send Rate", + "description": "###### What this is:\n*How often this node issues a ledger validation to the network. Each validation asserts the node fully validated one ledger.*\n\n###### How it's computed:\n*Per-second rate of validation events sent, summed per node over a 5-minute window.*\n\n###### Reading it:\n*A flat line at the network's ledger cadence is normal; higher is not better and lower means the node is validating less often.*\n\n###### Healthy range:\n*Roughly one validation per closed ledger (about 0.2-0.3/s on a healthy chain).*\n\n###### Watch for:\n*A drop to zero on a validator means it stopped validating; a value well below the close rate means validation is lagging.*\n\n###### Keywords:\n- **Ledger validation** *(network event)* \u2014 the second consensus stage where the node confirms a built ledger matches the trusted validator quorum and marks it final.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::createValidationSpan`\n\n###### References:\n[Ledger validation](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-validation)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.validation.send\"}[$__rate_interval])), \"series\", \"Validations / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + }, + "id": 1 + }, + { + "title": "Consensus Mode Over Time", + "description": "###### What this is:\n*Ledger-close activity split by the node's consensus mode (Proposing, Observing, Wrong Ledger, Switched Ledger).*\n\n###### How it's computed:\n*Per-second rate of ledger-close events grouped by consensus mode, per node, over a 5-minute window.*\n\n###### Reading it:\n*One dominant band is normal; a validator should sit almost entirely in Proposing, a stock node in Observing.*\n\n###### Healthy range:\n*Nearly all activity in a single expected mode.*\n\n###### Watch for:\n*Sustained time in Wrong Ledger or Switched Ledger indicates the node is out of sync or flapping between chains.*\n\n###### Keywords:\n- **Ledger close** *(network event)* \u2014 the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n- **Consensus mode** *(per node)* \u2014 the node's role/health in the current round: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::onClose`\n\n###### References:\n[Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus mode](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-close)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (consensus_mode, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.ledger_close\"}[$__rate_interval])), \"series\", \"$1\", \"consensus_mode\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Events / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 2 + }, + { + "title": "Consensus Proposals Sent Rate", + "description": "###### What this is:\n*How often this node proposes its candidate transaction set to peers during consensus.*\n\n###### How it's computed:\n*Per-second rate of proposal-send events, summed per node over a 5-minute window.*\n\n###### Reading it:\n*A steady line tracking the ledger cadence is healthy; only proposing (validator) nodes show activity.*\n\n###### Healthy range:\n*Steady output in step with the close rate for a proposing node; zero for a non-proposing node.*\n\n###### Watch for:\n*A proposing validator dropping to zero, or erratic spikes suggesting repeated re-proposals within rounds.*\n\n###### Keywords:\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Consensus mode** *(per node)* \u2014 the node's role/health in the current round: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::propose`\n\n###### References:\n[Proposal](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#proposal)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.proposal.send\"}[$__rate_interval])), \"series\", \"Proposals / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Proposals / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 3 + }, + { + "title": "Accept vs Close Rate", + "description": "###### What this is:\n*Two rates side by side: ledgers accepted after consensus versus ledger closes initiated.*\n\n###### How it's computed:\n*Per-second rate of accept events and of close events, each summed per node over a 5-minute window.*\n\n###### Reading it:\n*The two lines should overlap; read any gap between them as closes that did not finish accepting.*\n\n###### Healthy range:\n*Both lines equal and tracking the network close cadence.*\n\n###### Watch for:\n*A persistent gap where closes exceed accepts points to consensus rounds failing or timing out before acceptance.*\n\n###### Keywords:\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Ledger close** *(network event)* \u2014 the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan / RCLConsensus::Adaptor::onClose`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.accept\"}[$__rate_interval])), \"series\", \"Accepts / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.ledger_close\"}[$__rate_interval])), \"series\", \"Closes / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Events / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 4 + }, + { + "title": "Validation vs Close Rate", + "description": "###### What this is:\n*Two rates side by side: validations sent versus ledger closes, so you can see whether every closed ledger gets validated.*\n\n###### How it's computed:\n*Per-second rate of validation-send events and of close events, each summed per node over a 5-minute window.*\n\n###### Reading it:\n*The lines should overlap; a validation line below the close line means validation is falling behind.*\n\n###### Healthy range:\n*Both lines equal at the network close cadence.*\n\n###### Watch for:\n*Validations persistently trailing closes, which means the node validates fewer ledgers than it closes.*\n\n###### Keywords:\n- **Validations checked vs sent** *(per node)* \u2014 validations this node received from peers and checked, versus validations it issued itself.\n- **Ledger close** *(network event)* \u2014 the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::createValidationSpan / RCLConsensus::Adaptor::onClose`\n\n###### References:\n[Validations checked vs sent](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validations-checked-vs-sent)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.validation.send\"}[$__rate_interval])), \"series\", \"Validations / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.ledger_close\"}[$__rate_interval])), \"series\", \"Closes / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Events / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 5 + }, + { + "title": "Time to Reach Consensus (p50/p95)", + "description": "###### What this is:\n*Wall-clock time for the network to agree a ledger, at the median and 95th percentile.*\n\n###### How it's computed:\n*The recorded per-round agreement time is aggregated to its 50th and 95th percentile over the window.*\n\n###### Reading it:\n*Lower is better; the p95 line shows the worst typical rounds while p50 shows the common case.*\n\n###### Healthy range:\n*Roughly 2-4 seconds on mainnet; workload- and network-dependent.*\n\n###### Watch for:\n*Rising percentiles, or a p95 that pulls far above p50, signal slow or contentious rounds under load or poor peer connectivity.*\n\n###### Keywords:\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Convergence time** *(network event)* \u2014 the wall-clock time the network took to agree a ledger in a round.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan`\n\n###### References:\n[Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-round)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.round_time_ms, .5)", + "legendFormat": "P50 Time to Consensus [{{resource.service.instance.id}}]", + "refId": "A" + }, + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "refId": "B", + "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.round_time_ms, .95)", + "legendFormat": "P95 Time to Consensus [{{resource.service.instance.id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Time to Consensus (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 6 + }, + { + "title": "Average Time to Reach Consensus", + "description": "###### What this is:\n*Mean wall-clock time for the network to agree a ledger.*\n\n###### How it's computed:\n*The recorded per-round agreement time averaged over the window.*\n\n###### Reading it:\n*Lower is better; watch the trend rather than any single point.*\n\n###### Healthy range:\n*Roughly 2-4 seconds on mainnet; workload-dependent.*\n\n###### Watch for:\n*A steady upward drift indicates the network is taking longer to converge, often from load or connectivity problems.*\n\n###### Keywords:\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Convergence time** *(network event)* \u2014 the wall-clock time the network took to agree a ledger in a round.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan`\n\n###### References:\n[Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-round)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.round_time_ms)", + "legendFormat": "Avg Time to Consensus [{{resource.service.instance.id}}]", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Time to Consensus (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 7 + }, + { + "title": "Consensus Rounds per Ledger (Establish Count)", + "description": "###### What this is:\n*How many establish-phase iterations each ledger needed before validators converged.*\n\n###### How it's computed:\n*Ledgers are counted over the window and grouped by their iteration count; each series is one count value.*\n\n###### Reading it:\n*Most weight on the '1' series is ideal; weight shifting to 2, 3 or more shows harder convergence.*\n\n###### Healthy range:\n*The large majority of ledgers converging in one iteration.*\n\n###### Watch for:\n*A growing share of ledgers needing several iterations, indicating disagreement or network stress.*\n\n###### Keywords:\n- **Establish phase** *(network event)* \u2014 the consensus phase where validators iterate proposals; the establish count is how many iterations a ledger needed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/consensus/Consensus.h)\n\n###### Function:\n`Consensus::startEstablishTracing`\n\n###### References:\n[Establish phase](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#establish-phase)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max", "lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "refId": "A", + "query": "{name=\"consensus.establish\" && resource.service.instance.id=~\"$node\"} | count_over_time() by (span.establish_count)", + "legendFormat": "{{span.establish_count}} Rounds" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "axisLabel": "Ledgers in Window", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "decimals": 0 + }, + "overrides": [] + }, + "id": 8 + }, + { + "title": "Previous Round Time per Ledger", + "description": "###### What this is:\n*Duration of the immediately preceding consensus round, at the 95th percentile.*\n\n###### How it's computed:\n*The prior round's recorded duration is aggregated to its 95th percentile over the window.*\n\n###### Reading it:\n*Lower is better; complements the time-to-consensus panels with the previous round's cost.*\n\n###### Healthy range:\n*Comparable to the current time-to-consensus, roughly a few seconds.*\n\n###### Watch for:\n*A rising p95 means recent rounds have been expensive, often preceding a slowdown in ledger cadence.*\n\n###### Keywords:\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Convergence time** *(network event)* \u2014 the wall-clock time the network took to agree a ledger in a round.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::startRoundTracing`\n\n###### References:\n[Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-round)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.round\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.previous_round_time_ms, .95)", + "legendFormat": "P95 Previous Round [{{resource.service.instance.id}}]", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Round Time (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 9 + }, + { + "title": "Position Update Duration", + "description": "###### What this is:\n*Time this node spends each round tallying disputes and updating its consensus position, at the 95th percentile.*\n\n###### How it's computed:\n*Per-round position-update durations are aggregated to their 95th percentile over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; short, flat durations are normal.*\n\n###### Healthy range:\n*A few milliseconds under normal load.*\n\n###### Watch for:\n*Sustained high durations point to heavy dispute resolution or slow convergence on close time.*\n\n###### Keywords:\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Position update** *(per node)* \u2014 the per-round work of tallying disputes and revising the node's own consensus position.\n- **Close time** *(network event)* \u2014 the timestamp validators agree to stamp on a ledger, rounded to a shared resolution.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/consensus/Consensus.h)\n\n###### Function:\n`Consensus::updateOurPositions`\n\n###### References:\n[Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Close time](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-round)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.update_positions\"}[5m]))), \"series\", \"P95 Update\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 10 + }, + { + "title": "Ledger Close Duration", + "description": "###### What this is:\n*Full consensus round duration (open to accept) at the 95th percentile — the time a ledger takes to close.*\n\n###### How it's computed:\n*95th percentile of the consensus.round span duration per node over a 5-minute window.*\n\n###### Reading it:\n*Tracks the network close interval (~3-5s on mainnet); lower and steadier is better.*\n\n###### Healthy range:\n*A few seconds, matching the close cadence; workload-dependent on test networks.*\n\n###### Watch for:\n*A rising p95 means rounds are taking longer to converge.*\n\n###### Note:\n*Uses consensus.round, not consensus.ledger_close: the latter span only wraps the sub-millisecond onClose() prologue and is not the ledger close time.*\n\n###### Keywords:\n- **Consensus round** *(network event)* — one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Consensus** *(network event)* — the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Ledger close interval** *(network-wide)* — the network's steady ledger rhythm — roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Convergence time** *(network event)* — the wall-clock time the network took to agree a ledger in a round.\n- **Ledger close** *(network event)* — the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::onClose (round span)`\n\n###### References:\n[Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) · [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) · [Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-round)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.round\"}[5m]))), \"series\", \"P95 Close (consensus.round)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 11 + }, + { + "title": "Ledger Apply Duration (doAccept)", + "description": "###### What this is:\n*Time spent applying the agreed transaction set to build the new ledger, at the 95th percentile.*\n\n###### How it's computed:\n*Per-apply durations are aggregated to their 95th percentile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; this is the cost of executing the ledger's transactions.*\n\n###### Healthy range:\n*A few to tens of milliseconds, scaling with the number of transactions in the ledger.*\n\n###### Watch for:\n*A rising p95 alongside high transaction counts signals apply-stage load, whether organic or from a flood of transactions.*\n\n###### Keywords:\n- **Transaction apply phase** *(per node)* \u2014 the step that executes the agreed transaction set into the new ledger during a close.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Transaction apply phase](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-apply-phase)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.accept.apply\"}[5m]))), \"series\", \"P95 Apply Duration\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 12 + }, + { + "title": "Consensus Accept Duration Heatmap", + "description": "###### What this is:\n*Distribution of how long the accept step takes across rounds, over time.*\n\n###### How it's computed:\n*Accept durations are bucketed by duration band and counted per 5-minute window, drawn as a heatmap.*\n\n###### Reading it:\n*A tight low band is healthy; brighter cells higher up mean more slow rounds.*\n\n###### Healthy range:\n*Most rounds concentrated in the lowest duration bands.*\n\n###### Watch for:\n*A widening or upward-drifting hot band flags outlier rounds that take abnormally long.*\n\n###### Keywords:\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan`\n\n###### References:\n[Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-round)", + "type": "heatmap", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "yAxis": { + "axisLabel": "Duration (ms)", + "unit": "ms" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.accept\"}[5m])) by (le)", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + } + }, + "id": 13 + }, + { + "title": "Close-Time Agreement Rate (Agreed vs Disagreed)", + "description": "###### What this is:\n*How often validators agreed on the ledger close time versus agreed to disagree.*\n\n###### How it's computed:\n*Per-second rate of rounds split into Agreed and Disagreed, per node, over the rate interval.*\n\n###### Reading it:\n*Mostly Agreed is healthy; occasional Disagreed is normal when validator clocks differ slightly.*\n\n###### Healthy range:\n*Overwhelmingly Agreed.*\n\n###### Watch for:\n*A sustained rise in Disagreed points to clock drift or latency spread across the validator set.*\n\n###### Keywords:\n- **Ledger close** *(network event)* \u2014 the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Clock drift** *(network event)* \u2014 differences between validators' clocks, which spread their proposed close times.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Clock drift](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-close)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (close_time_correct, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{span_name=\"consensus.accept.apply\", consensus_mode=~\"$consensus_mode\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"$1\", \"close_time_correct\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Rounds / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 14 + }, + { + "title": "Close Time Vote Bins & Resolution", + "description": "###### What this is:\n*Two related values: how many distinct close-time votes were seen, and the current close-time rounding granularity in ms.*\n\n###### How it's computed:\n*Both values are averaged over the window and plotted on dual axes.*\n\n###### Reading it:\n*Fewer vote bins and a fine resolution mean tight agreement; more bins and a coarse resolution mean disagreement.*\n\n###### Healthy range:\n*Few bins with a fine (about 10s) resolution.*\n\n###### Watch for:\n*Rising bins with the resolution widening (toward 120s) shows validators struggling to agree on close time.*\n\n###### Keywords:\n- **Distinct positions** *(network event)* \u2014 the number of different close-time values validators proposed in a round; one means full agreement.\n- **Close-time resolution** *(network event)* \u2014 the granularity (in seconds) that close times are rounded to; widens when validators disagree.\n- **Close time** *(network event)* \u2014 the timestamp validators agree to stamp on a ledger, rounded to a shared resolution.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Distinct positions](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#distinct-positions)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "pointSize": 5, + "showPoints": "auto", + "axisLabel": "Count / Milliseconds", + "spanNulls": 1800000 + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Vote Bins" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Resolution" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.close_time_vote_bins)", + "legendFormat": "Avg Vote Bins", + "refId": "A" + }, + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.close_resolution_ms)", + "legendFormat": "Avg Resolution (ms)", + "refId": "B" + } + ], + "id": 15 + }, + { + "title": "Close-Time Resolution Change (per Round)", + "description": "###### What this is:\n*Whether the close-time rounding granularity moved Coarser, Finer, or stayed Steady versus the previous round.*\n\n###### How it's computed:\n*Rounds are counted over the window and grouped by direction of change.*\n\n###### Reading it:\n*Mostly Steady is healthy; Coarser means widening bins (more disagreement), Finer means tightening.*\n\n###### Healthy range:\n*Predominantly Steady.*\n\n###### Watch for:\n*Frequent Coarser shifts indicate the network is repeatedly failing to agree on close time.*\n\n###### Keywords:\n- **Close-time resolution** *(network event)* \u2014 the granularity (in seconds) that close times are rounded to; widens when validators disagree.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Close time** *(network event)* \u2014 the timestamp validators agree to stamp on a ledger, rounded to a shared resolution.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Close-time resolution](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#close-time-resolution)", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "legend": { + "showLegend": false + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)", + "legendFormat": "{{span.resolution_direction}}", + "refId": "A" + } + ], + "id": 16, + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "increased", + "renamePattern": "Coarser (more disagreement)" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "decreased", + "renamePattern": "Finer (better agreement)" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "unchanged", + "renamePattern": "Steady" + } + } + ] + }, + { + "title": "Close-Time Proposal Spread (Distinct Positions per Round)", + "description": "###### What this is:\n*How spread out validators' proposed close times were, as the number of distinct positions per round.*\n\n###### How it's computed:\n*Rounds are counted over the window and grouped by their distinct-position count; each bar is one count value.*\n\n###### Reading it:\n*Weight on '1 distinct position' means everyone agreed; weight on 2 or more means the proposals split.*\n\n###### Healthy range:\n*Most rounds at a single distinct position.*\n\n###### Watch for:\n*A shift toward two or more distinct positions signals growing clock drift or latency across validators.*\n\n###### Keywords:\n- **Distinct positions** *(network event)* \u2014 the number of different close-time values validators proposed in a round; one means full agreement.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Clock drift** *(network event)* \u2014 differences between validators' clocks, which spread their proposed close times.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Distinct positions](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#distinct-positions)", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "legend": { + "showLegend": false + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | count_over_time() by (span.close_time_vote_bins)", + "legendFormat": "{{span.close_time_vote_bins}} Distinct Position(s)", + "refId": "A" + } + ], + "id": 17 + }, + { + "title": "Consensus Outcome Distribution (per Round)", + "description": "###### What this is:\n*How consensus rounds concluded: Agreed, Moved On (partial), Expired (timeout), or No Consensus.*\n\n###### How it's computed:\n*Rounds over the window are counted and grouped by outcome, shown as shares of a pie.*\n\n###### Reading it:\n*An almost fully Agreed pie is healthy; other slices represent degraded outcomes.*\n\n###### Healthy range:\n*Nearly 100 percent Agreed.*\n\n###### Watch for:\n*A growing Moved On or Expired share signals network stress, disagreement, or connectivity loss.*\n\n###### Keywords:\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Consensus outcome** *(network event)* \u2014 how a round concluded: Agreed, Moved On (partial), Expired (timeout), or No Consensus.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan`\n\n###### References:\n[Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Consensus outcome](https://xrpl.org/docs/concepts/consensus-protocol/consensus-principles-and-rules) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-round)", + "type": "piechart", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 80 + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "values": ["value", "percent"] + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, consensus_state, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.accept\", consensus_state!=\"\"}[$__rate_interval])), \"series\", \"$1\", \"consensus_state\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short" + }, + "overrides": [] + }, + "id": 18 + }, + { + "title": "Consensus Failures Over Time", + "description": "###### What this is:\n*Rate of non-normal consensus outcomes (moved-on plus expired) over time.*\n\n###### How it's computed:\n*Per-second rate of the moved-on and expired outcomes, summed per node over a 5-minute window.*\n\n###### Reading it:\n*Flat at zero is healthy; any sustained line is a concern.*\n\n###### Healthy range:\n*Zero under normal operation.*\n\n###### Watch for:\n*Spikes or a persistent nonzero rate indicate consensus instability that can precede ledger stalls or forks.*\n\n###### Keywords:\n- **Consensus outcome** *(network event)* \u2014 how a round concluded: Agreed, Moved On (partial), Expired (timeout), or No Consensus.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan`\n\n###### References:\n[Consensus outcome](https://xrpl.org/docs/concepts/consensus-protocol/consensus-principles-and-rules) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-outcome)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 88 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.accept\", consensus_state=\"moved_on\"}[$__rate_interval])), \"series\", \"moved_on\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.accept\", consensus_state=\"expired\"}[$__rate_interval])), \"series\", \"expired\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Failures / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 19 + }, + { + "title": "Consensus Stall Rate", + "description": "###### What this is:\n*Rate at which consensus health checks report a stalled condition versus not stalled.*\n\n###### How it's computed:\n*Per-second rate of consensus checks split by stalled flag, summed per node over a 5-minute window.*\n\n###### Reading it:\n*The Not Stalled line should carry all the weight; any Stalled line is an early warning.*\n\n###### Healthy range:\n*Stalled at zero.*\n\n###### Watch for:\n*A nonzero stalled rate surfaces stalls before they show up as validated-ledger-age alarms.*\n\n###### Keywords:\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Consensus stall** *(per node)* \u2014 a health check reporting that consensus is not making forward progress.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/consensus/Consensus.h)\n\n###### Function:\n`Consensus::haveConsensus`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Consensus stall](https://xrpl.org/docs/concepts/consensus-protocol/consensus-principles-and-rules) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 88 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.check\", consensus_stalled=\"true\"}[$__rate_interval])), \"series\", \"Stalled\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.check\", consensus_stalled=\"false\"}[$__rate_interval])), \"series\", \"Not Stalled\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Checks / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 20 + }, + { + "title": "Consensus Mode-Change Rate by Target Mode", + "description": "###### What this is:\n*How often the node switches consensus mode, broken down by the mode it switched into.*\n\n###### How it's computed:\n*Per-second rate of mode-change events grouped by target mode, per node, over a 5-minute window.*\n\n###### Reading it:\n*Near-zero is healthy; a stable node rarely changes mode.*\n\n###### Healthy range:\n*Essentially zero mode changes in steady state.*\n\n###### Watch for:\n*Frequent switches into Wrong Ledger or Switched Ledger mark an unstable node at risk of forking.*\n\n###### Keywords:\n- **Consensus mode** *(per node)* \u2014 the node's role/health in the current round: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n- **Fork** *(network-wide)* \u2014 when nodes validate divergent ledger chains instead of a single agreed history.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::onModeChange`\n\n###### References:\n[Consensus mode](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Fork](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-mode)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 96 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (mode_new, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.mode_change\"}[$__rate_interval])), \"series\", \"$1\", \"mode_new\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Mode Changes / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 21 + }, + { + "title": "Ledger History Mismatch Rate by Reason", + "description": "###### What this is:\n*Rate of built-versus-validated ledger mismatches, broken down by why they diverged.*\n\n###### How it's computed:\n*Per-second rate of mismatch events grouped by reason, per node, over a 5-minute window.*\n\n###### Reading it:\n*Flat at zero is healthy; the reason label tells you the nature of any divergence.*\n\n###### Healthy range:\n*Zero under normal operation.*\n\n###### Watch for:\n*Any sustained rate marks a fork; the reason distinguishes close-time disagreement, sync drift, and transaction-processing differences.*\n\n###### Keywords:\n- **Ledger history mismatch** *(per node)* \u2014 when a locally built ledger's hash does not match the network-validated hash.\n- **Fork** *(network-wide)* \u2014 when nodes validate divergent ledger chains instead of a single agreed history.\n- **Close time** *(network event)* \u2014 the timestamp validators agree to stamp on a ledger, rounded to a shared resolution.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementLedgerHistoryMismatch`\n\n###### References:\n[Fork](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Close time](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-history-mismatch)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 96 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_node_role) (rate(ledger_history_mismatch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"$1\", \"reason\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Mismatches / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 22 + } + ], + "schemaVersion": 39, + "tags": ["consensus"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "DS_TEMPO", + "type": "datasource", + "label": "Tempo", + "query": "tempo", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "consensus_mode", + "label": "Consensus Mode", + "description": "Filter by consensus mode (Proposing, Observing, Wrong Ledger, Switched Ledger)", + "type": "query", + "query": "label_values(span_calls_total{span_name=\"consensus.ledger_close\"}, consensus_mode)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "close_time_correct", + "label": "Close Time Agreed", + "type": "custom", + "query": "true,false", + "current": { + "text": "All", + "value": "$__all" + }, + "includeAll": true, + "allValue": ".*", + "multi": true, + "options": [ + { + "text": "All", + "value": "$__all", + "selected": true + }, + { + "text": "true", + "value": "true", + "selected": false + }, + { + "text": "false", + "value": "false", + "selected": false + } + ] + }, + { + "name": "resolution_direction", + "label": "Resolution Direction", + "type": "custom", + "query": "increased,decreased,unchanged", + "current": { + "text": "All", + "value": "$__all" + }, + "includeAll": true, + "allValue": ".*", + "multi": true, + "options": [ + { + "text": "All", + "value": "$__all", + "selected": true + }, + { + "text": "increased", + "value": "increased", + "selected": false + }, + { + "text": "decreased", + "value": "decreased", + "selected": false + }, + { + "text": "unchanged", + "value": "unchanged", + "selected": false + } + ] + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Consensus Health", + "uid": "consensus-health", + "description": "What this shows: Consensus health for XRPL nodes: how reliably and quickly the network agrees each ledger, and where agreement breaks down. \u2014 Use it to: Spot stalled or slow consensus rounds and pinpoint the phase where agreement is failing.", + "refresh": "30s" +} diff --git a/docker/telemetry/grafana/dashboards/fee-market.json b/docker/telemetry/grafana/dashboards/fee-market.json new file mode 100644 index 00000000000..df3db01c98b --- /dev/null +++ b/docker/telemetry/grafana/dashboards/fee-market.json @@ -0,0 +1,604 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Transaction queue depth and capacity, fee-escalation levels, and load-factor breakdown that together set the minimum fee to enter the open ledger. \u2014 Use it to: Understand why the minimum transaction fee is rising and how full the open ledger and queue are.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Transaction Queue Depth", + "description": "###### What this is:\n*Transactions currently waiting in the transaction queue versus the queue's maximum capacity.*\n\n###### How it's computed:\n*Instantaneous gauge readings of current queue count and configured max size.*\n\n###### Reading it:\n*Queue depth well below capacity is normal; depth approaching capacity means the node is saturating.*\n\n###### Healthy range:\n*Depth near 0 in quiet periods; workload-dependent under load.*\n\n###### Watch for:\n*Depth pinned at capacity for sustained periods, which signals demand exceeding throughput or a fee-spam burst.*\n\n###### Keywords:\n- **Transaction queue (TxQ)** *(per node)* \u2014 holds transactions that meet local cost but not the open-ledger cost, to include in a later ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerTxqGauge`\n\n###### References:\n[Transaction queue (TxQ)](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-queue-txq)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_count\"}, \"series\", \"Queue Depth\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_max_size\"}, \"series\", \"Max Capacity\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Transactions", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "Transactions Per Ledger", + "description": "###### What this is:\n*Transactions already placed in the current open ledger versus the expected per-ledger target.*\n\n###### How it's computed:\n*Instantaneous gauge readings of in-ledger count and the target count that governs fee escalation.*\n\n###### Reading it:\n*Staying at or below the expected target is normal; exceeding it triggers open-ledger fee escalation.*\n\n###### Healthy range:\n*At or under the expected per-ledger target.*\n\n###### Watch for:\n*In-ledger count persistently above target, indicating sustained congestion pushing fees up.*\n\n###### Keywords:\n- **Open ledger** *(per node)* \u2014 the temporary workspace ledger where incoming transactions are provisionally applied before a close.\n- **In-ledger vs target count** *(per node)* \u2014 transactions already in the open ledger versus the soft target that triggers fee escalation.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerTxqGauge`\n\n###### References:\n[Open ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [In-ledger vs target count](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#open-ledger)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_in_ledger\"}, \"series\", \"In Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_per_ledger\"}, \"series\", \"Expected Per Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Transactions", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "Fee Escalation Levels", + "description": "###### What this is:\n*The fee levels that govern queue admission: reference (baseline), minimum processing, median, and open-ledger levels.*\n\n###### How it's computed:\n*Instantaneous gauge readings of each fee level, shown on a log scale.*\n\n###### Reading it:\n*Open-ledger level near the reference level means cheap entry; a large gap above reference means escalation is active.*\n\n###### Healthy range:\n*Open-ledger level at or near reference during normal traffic.*\n\n###### Watch for:\n*Open-ledger level spiking far above reference, the hallmark of congestion or a fee-bidding war.*\n\n###### Keywords:\n- **Queue admission rejection** *(per node)* \u2014 a transaction refused entry to the queue, e.g. queue_full when the queue is at capacity.\n- **Fee levels** *(per node)* \u2014 cost thresholds governing queue admission: reference (baseline), minimum, median, and open-ledger.\n- **Open ledger** *(per node)* \u2014 the temporary workspace ledger where incoming transactions are provisionally applied before a close.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerTxqGauge`\n\n###### References:\n[Queue admission rejection](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Fee levels](https://xrpl.org/docs/concepts/transactions/transaction-cost#fee-levels) \u00b7 [Open ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Fee escalation](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#queue-admission-rejection)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_reference_fee_level\"}, \"series\", \"Reference Fee Level\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_min_processing_fee_level\"}, \"series\", \"Min Processing Fee Level\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_med_fee_level\"}, \"series\", \"Median Fee Level\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_open_ledger_fee_level\"}, \"series\", \"Open Ledger Fee Level\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Fee Level", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "scaleDistribution": { + "type": "log", + "log": 2 + }, + "spanNulls": 1800000 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "Load Factor Breakdown", + "description": "###### What this is:\n*The combined load factor and its server, fee-escalation, and fee-queue contributors as unitless fee multipliers (1.0 = no load).*\n\n###### How it's computed:\n*Instantaneous gauge readings of each load-factor component.*\n\n###### Reading it:\n*Values at 1.0 mean base fees; higher values raise the fee to transact.*\n\n###### Healthy range:\n*Around 1.0 under normal conditions.*\n\n###### Watch for:\n*Combined factor climbing well above 1.0, showing the node is charging premium fees due to congestion or overload.*\n\n###### Keywords:\n- **Load factor** *(per node)* \u2014 a unitless multiplier (1.0 = no load) that scales the base transaction cost as the node comes under load.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n- **Transaction cost** *(network-wide)* \u2014 the XRP a transaction destroys to be processed; scales up with load to deter spam.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLoadFactorGauge`\n\n###### References:\n[Load factor](https://xrpl.org/docs/concepts/transactions/transaction-cost#local-load-cost) \u00b7 [Fee escalation](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Transaction cost](https://xrpl.org/docs/concepts/transactions/transaction-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#load-factor)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor\"}, \"series\", \"Combined Load Factor\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_server\"}, \"series\", \"Server\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_fee_escalation\"}, \"series\", \"Fee Escalation\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_fee_queue\"}, \"series\", \"Fee Queue\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Multiplier", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "spanNulls": 1800000 + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + } + }, + { + "title": "Load Factor Components", + "description": "###### What this is:\n*The individual load-factor inputs, local server load, network load, and cluster load, as unitless multipliers.*\n\n###### How it's computed:\n*Instantaneous gauge readings of each component.*\n\n###### Reading it:\n*All at 1.0 means no load pressure from any source; a raised component identifies where load originates.*\n\n###### Healthy range:\n*Around 1.0 for each component.*\n\n###### Watch for:\n*A single component rising sharply, which pinpoints whether the pressure is local, network-wide, or cluster-driven.*\n\n###### Keywords:\n- **Load factor** *(per node)* \u2014 a unitless multiplier (1.0 = no load) that scales the base transaction cost as the node comes under load.\n- **Cluster** *(cluster-wide)* \u2014 a group of trusted co-operated nodes that share load information and skip some verification.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLoadFactorGauge`\n\n###### References:\n[Load factor](https://xrpl.org/docs/concepts/transactions/transaction-cost#local-load-cost) \u00b7 [Cluster](https://xrpl.org/docs/concepts/networks-and-servers/clustering) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#load-factor)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_local\"}, \"series\", \"Local\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_net\"}, \"series\", \"Network\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_cluster\"}, \"series\", \"Cluster\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Multiplier", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "spanNulls": 1800000 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "Queue Abandonment Rate (Expired)", + "description": "###### What this is:\n*Transactions dropped from the queue because their last-ledger deadline passed before they could be included.*\n\n###### How it's computed:\n*Per-second rate of the cumulative expired-transaction counter over the dashboard's rate interval.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means submitters under-bid the escalating fee and their transactions timed out.*\n\n###### Healthy range:\n*Near 0 expirations per second.*\n\n###### Watch for:\n*Sustained expiry rate, a demand-frustration signal often coinciding with fee spikes or spam that crowds out honest traffic.*\n\n###### Keywords:\n- **Queue admission rejection** *(per node)* \u2014 a transaction refused entry to the queue, e.g. queue_full when the queue is at capacity.\n- **Queue expiry / abandonment** *(per node)* \u2014 removing a queued transaction whose LastLedgerSequence deadline passed before inclusion.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementTxqExpired (caller TxQ.cpp)`\n\n###### References:\n[Queue admission rejection](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Queue expiry / abandonment](https://xrpl.org/docs/concepts/transactions/reliable-transaction-submission) \u00b7 [Fee escalation](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#queue-admission-rejection)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(txq_expired_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Expired / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Expired / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Queue Admission Rejections (Dropped)", + "description": "###### What this is:\n*Transactions refused entry to the queue, broken down by reason such as queue_full.*\n\n###### How it's computed:\n*Per-second rate of the cumulative dropped-transaction counter over the dashboard's rate interval, split by reason.*\n\n###### Reading it:\n*Near zero is healthy; queue_full rejections mean the queue is at capacity and applying backpressure.*\n\n###### Healthy range:\n*Near 0 rejections per second.*\n\n###### Watch for:\n*A burst of queue_full drops, distinct from expiry, indicating the node is being flooded faster than it can drain.*\n\n###### Keywords:\n- **Queue admission rejection** *(per node)* \u2014 a transaction refused entry to the queue, e.g. queue_full when the queue is at capacity.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementTxqDropped (caller TxQ.cpp)`\n\n###### References:\n[Queue admission rejection](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#queue-admission-rejection)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_node_role) (rate(txq_dropped_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"$1\", \"reason\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Dropped / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["transactions", "fees"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Fee Market & TxQ", + "uid": "fee-market", + "version": 1, + "refresh": "30s" +} diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json new file mode 100644 index 00000000000..2ffbf8e0b1d --- /dev/null +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -0,0 +1,654 @@ +{ + "annotations": { + "list": [] + }, + "description": "What this shows: Per-job-type throughput, queue wait times, and execution times for the node's internal worker job queue.\nUse it to: Find job types that are backing up or running slowly and causing internal processing delays.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Current Job Latency (p99 Gauge)", + "description": "###### What this is:\n*At-a-glance p99 of how long jobs wait in the queue and how long they run once started.*\n\n###### How it's computed:\n*99th percentile derived from the job wait-time and run-time histograms over the last 5 minutes.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Wait and exec p99 under 100ms.*\n\n###### Watch for:\n*p99 wait climbing into the red, meaning worker threads are saturated and jobs are backing up.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted / recordJobFinished`", + "type": "gauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(histogram_quantile(0.99, sum by (le, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(histogram_quantile(0.99, sum by (le, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "µs", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100000 + }, + { + "color": "red", + "value": 1000000 + } + ] + } + }, + "overrides": [] + }, + "id": 1 + }, + { + "title": "Job Throughput Rate (Per Second)", + "description": "###### What this is:\n*Rate of jobs queued, started, and finished across all job types.*\n\n###### How it's computed:\n*Per-second rate of each cumulative job counter over a 5-minute window.*\n\n###### Reading it:\n*Queued, started, and finished tracking together means the queue keeps up.*\n\n###### Healthy range:\n*Workload-dependent; the three rates should stay roughly equal.*\n\n###### Watch for:\n*Queued rate persistently above finished rate, which indicates a growing backlog.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobQueued / recordJobStarted / recordJobFinished`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval])), \"series\", \"Queued/s\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval])), \"series\", \"Started/s\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval])), \"series\", \"Finished/s\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 2 + }, + { + "title": "Per-Job-Type Queued Rate", + "description": "###### What this is:\n*Rate of jobs entering the queue, broken down by job type (top 10).*\n\n###### How it's computed:\n*Per-second rate of the queued-job counter per job_type over a 5-minute window.*\n\n###### Reading it:\n*Identifies which job types generate the most queue activity.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A single job type surging unexpectedly, which can point to a flood of a particular request or peer message.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobQueued`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(topk(10, sum by (job_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval]))), \"series\", \"$1\", \"job_type\", \"(.*)\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 3 + }, + { + "title": "Per-Job-Type Finish Rate", + "description": "###### What this is:\n*Rate of jobs completing, broken down by job type (top 10).*\n\n###### How it's computed:\n*Per-second rate of the finished-job counter per job_type over a 5-minute window.*\n\n###### Reading it:\n*Compare against the queued rate per type to spot which types are falling behind.*\n\n###### Healthy range:\n*Workload-dependent; should match the per-type queued rate.*\n\n###### Watch for:\n*A type whose finish rate lags its queued rate, revealing where the backlog concentrates.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(topk(10, sum by (job_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval]))), \"series\", \"$1\", \"job_type\", \"(.*)\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 4 + }, + { + "title": "Job Queue Wait Time", + "description": "###### What this is:\n*Distribution of how long jobs sit in the queue before a worker picks them up (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job wait-time histogram over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; a widening gap between p75 and p99 signals occasional stalls.*\n\n###### Healthy range:\n*Sub-millisecond to low-millisecond waits on an unloaded node.*\n\n###### Watch for:\n*Rising p99 wait, an early sign of worker-thread saturation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(histogram_quantile(0.75, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p75 Wait\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "µs", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "axisLabel": "Duration (μs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 5 + }, + { + "title": "Job Execution Time", + "description": "###### What this is:\n*Distribution of how long jobs run once started (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job run-time histogram over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; stable p75 with a controlled p99 is healthy.*\n\n###### Healthy range:\n*Workload-dependent, but stable over time.*\n\n###### Watch for:\n*Growing execution times, which point to CPU pressure or expensive individual jobs.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(histogram_quantile(0.75, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p75 Exec\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "µs", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "axisLabel": "Duration (μs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 6 + }, + { + "title": "Per-Job-Type Execution Time (p99)", + "description": "###### What this is:\n*The 10 slowest job types ranked by p99 execution time.*\n\n###### How it's computed:\n*p99 derived from the run-time histogram per job_type over a 5-minute window, top 10 selected.*\n\n###### Reading it:\n*Highlights which job types cost the most CPU time at the tail.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A job type whose p99 grows over time, indicating a slow or degrading operation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"$1\", \"job_type\", \"(.*)\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "µs", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Duration (μs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 7 + }, + { + "title": "Transaction Overflow Rate", + "description": "###### What this is:\n*Rate at which transaction jobs are shed when the queue's transaction limit is exceeded.*\n\n###### How it's computed:\n*Per-second rate of the overflow counter over the dashboard's rate interval, scaled to per minute. The counter is observed from the overlay's cumulative overflow tally.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means the job queue is shedding transaction work under load.*\n\n###### Healthy range:\n*0 overflows per minute.*\n\n###### Watch for:\n*Any sustained non-zero rate — the node is dropping transaction jobs because the queue is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerParityCounters (observed from Overlay::getJqTransOverflow)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(jq_trans_overflow_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Overflows/min\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "cpm", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "custom": { + "axisLabel": "Overflows / Min", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 8 + } + ], + "schemaVersion": 39, + "tags": ["node", "jobqueue"], + "templating": { + "list": [ + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "job_type", + "label": "Job Type", + "description": "Filter by job type", + "type": "query", + "query": "label_values(job_queued_total, job_type)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "handler", + "label": "Handler", + "description": "Filter by enqueueing handler (sanitised addJob name, e.g. RcvGetLedger)", + "type": "query", + "query": "label_values(job_queued_total, handler)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Job Queue Analysis", + "uid": "job-queue", + "version": 1, + "refresh": "5s" +} diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json new file mode 100644 index 00000000000..b052e03286a --- /dev/null +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -0,0 +1,1642 @@ +{ + "annotations": { + "list": [] + }, + "description": "What this shows: Ledger data exchange and object-fetch traffic between this node and its peers: ledger sync, tree-node retrieval, and transaction-set exchange.\nUse it to: See how much ledger data the node is pulling or serving and spot catch-up activity.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Ledger Data — Ledger", + "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_data_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Data Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_data_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Data Share\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 1 + }, + { + "title": "Ledger Data — Transaction", + "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_data_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_data_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Share\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_data_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_data_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Share\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 2 + }, + { + "title": "Ledger Data — Account State", + "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_data_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Node Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_data_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Node Share\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 3 + }, + { + "title": "Ledger Traffic — Ledger", + "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Get In\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Share In\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 4 + }, + { + "title": "Ledger Traffic — Transaction", + "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Share\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Share\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 5 + }, + { + "title": "Ledger Traffic — Account State", + "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(ledger_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Share\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 6 + }, + { + "title": "GetObject — Ledger", + "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Share\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 7 + }, + { + "title": "GetObject — Transaction", + "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_transaction_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transaction Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_transaction_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transaction Share\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Share\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 8 + }, + { + "title": "GetObject — Account State", + "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Share\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 9 + }, + { + "title": "GetObject Messages — Ledger", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_ledger_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Get\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "cps", + "custom": { + "axisLabel": "Messages In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 10 + }, + { + "title": "GetObject Messages — Transaction", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_transaction_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transaction Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_transaction_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "cps", + "custom": { + "axisLabel": "Messages In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 11 + }, + { + "title": "GetObject Messages — Account State", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_account_state_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Get\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "cps", + "custom": { + "axisLabel": "Messages In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 12 + }, + { + "title": "GetObject Messages — Specials", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_cas_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"CAS Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_fetch_pack_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetch Pack Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_transactions_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transactions Get\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "cps", + "custom": { + "axisLabel": "Messages In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 13 + }, + { + "title": "GetObject — Specials", + "description": "###### What this is:\n*Aggregate object-fetch inbound bytes plus special buckets: content-addressed storage fetches, bulk fetch-pack downloads used during catch-up, and bulk transaction fetches.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Fetch-pack rises sharply while catching up a range of ledgers; near zero when fully synced.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*Continuous fetch-pack traffic (node never fully catches up) or unexpectedly high content-store volume.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_cas_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"CAS Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_cas_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"CAS Share\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_fetch_pack_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetch Pack Share\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_fetch_pack_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetch Pack Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_transactions_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transactions Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Aggregate Get\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(getobject_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Aggregate Share\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 14 + }, + { + "title": "Overlay Traffic Heatmap (All Categories, Bytes In)", + "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "bargauge", + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 56 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1048576 + }, + { + "color": "red", + "value": 104857600 + } + ] + } + }, + "overrides": [] + }, + "id": 15 + }, + { + "title": "Sync Diagnostics", + "type": "row", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 16, + "panels": [] + }, + { + "title": "Sync State", + "description": "###### What this is:\n*Current server operating state as a numeric code: 0 disconnected, 1 connected, 2 syncing, 3 tracking, 4 full. A healthy synced node sits at 4.*\n\n###### How it's computed:\n*state_tracking{metric=\"state_value\"} (gauge). Companion time_in_current_state_seconds shows how long it has been stuck there.*\n\n###### Reading it:\n*Flat at 4 = full/healthy. Dropping to 1-2 and staying = the node fell out of sync and is re-acquiring (the primary red flag this row explains).*\n\n###### Healthy range:\n*4 (full), briefly 2-3 right after restart.*\n\n###### Watch for:\n*A node stuck below 4 for more than a few minutes, or oscillating - read the lower panels for the cause.*\n\n###### Source:\n[app/misc/NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_tracking{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"state_value\"}, \"series\", \"Server State\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Server State", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Disconnected", + "color": "red", + "index": 0 + }, + "1": { + "text": "Connected", + "color": "orange", + "index": 1 + }, + "2": { + "text": "Syncing", + "color": "yellow", + "index": 2 + }, + "3": { + "text": "Tracking", + "color": "blue", + "index": 3 + }, + "4": { + "text": "Full", + "color": "green", + "index": 4 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "blue", + "value": 3 + }, + { + "color": "green", + "value": 4 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "min": 0, + "max": 4 + }, + "overrides": [] + }, + "id": 17 + }, + { + "title": "Validated Ledger Age", + "description": "###### What this is:\n*Seconds since this node last had a freshly validated ledger. The single clearest 'am I keeping up with the network' signal.*\n\n###### How it's computed:\n*ledgermaster_validated_ledger_age gauge (seconds), per node.*\n\n###### Reading it:\n*Should hover at the network close interval (~3-5s). A rising sawtooth or a high plateau means the node is falling behind or not validating.*\n\n###### Healthy range:\n*<= ~6s on a synced node.*\n\n###### Watch for:\n*Sustained climb above ~15s, or a monotonic ramp = the node is not keeping up; correlate with job-queue wait and NuDB read latency below.*\n\n###### Source:\n[app/ledger/LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::getValidatedLedgerAge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Validated Age\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledgermaster_published_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Published Age\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "s", + "custom": { + "axisLabel": "Age (seconds)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 18 + }, + { + "title": "Ledger Close Rate", + "description": "###### What this is:\n*Rate at which the node closes ledgers, versus the ~0.25/s network cadence. A throughput deficit means it is not advancing in step with the network.*\n\n###### How it's computed:\n*rate(ledgers_closed_total[$__rate_interval]) per node.*\n\n###### Reading it:\n*Should track ~0.22-0.25 ledgers/s (one every ~4s). Near-zero while behind = stalled; a burst above network rate = catching up.*\n\n###### Healthy range:\n*~0.25/s steady on a synced node.*\n\n###### Watch for:\n*Near-zero close rate while Validated Ledger Age climbs = hard stall (e.g. genesis-flapping or disk-bound acquisition).*\n\n###### Source:\n[app/ledger/LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::closeLedger`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledgers_closed_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledgers Closed\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_fetches_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Fetches\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "reqps", + "custom": { + "axisLabel": "Rate (per second)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 19 + }, + { + "title": "Job Queue Wait p95 By Type", + "description": "###### What this is:\n*95th-percentile time a job waits in the queue before a worker thread picks it up, for the sync-critical job types. This is the metric form of the 'ProcessLData wait: NNNNms' warnings in the debug log.*\n\n###### How it's computed:\n*histogram_quantile(0.95, rate(jobq__q_milliseconds_bucket[$__rate_interval])) for ledgerdata, acceptledger, fetchtxndata, transaction, advanceledger, ledgerrequest.*\n\n###### Reading it:\n*Queue wait should be single-digit to low-tens of ms. High ledgerdata/fetchtxndata wait = the node cannot process inbound ledger data fast enough.*\n\n###### Healthy range:\n*< ~50ms p95 per type on a healthy node.*\n\n###### Watch for:\n*ledgerdata or fetchtxndata q-wait spiking to seconds = worker threads are blocked (usually on NuDB reads - see the cause tier).*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::getJson (per-type queue timing)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(jobq_ledgerdata_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerdata q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(jobq_acceptledger_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"acceptledger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(jobq_fetchtxndata_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"fetchtxndata q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(jobq_transaction_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"transaction q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(jobq_advanceledger_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"advanceledger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(jobq_ledgerrequest_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerrequest q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "p95 Wait (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 20 + }, + { + "title": "NuDB Read Latency", + "description": "###### What this is:\n*Average nodestore (NuDB) read latency: how long each on-disk object read takes. The direct disk-layer cost that backs up the job queue.*\n\n###### How it's computed:\n*rate(nodestore_state{metric=\"node_reads_duration_us\"}[$__rate_interval]) / rate(nodestore_state{metric=\"node_reads_total\"}[$__rate_interval]), in us.*\n\n###### Reading it:\n*Low single-digit us when the OS page cache is warm; tens-to-hundreds of us when reads hit the disk. Rises sharply during cold-cache catch-up.*\n\n###### Healthy range:\n*< ~10us/read warm; higher is expected briefly after a wipe/restart.*\n\n###### Watch for:\n*Sustained high us/read together with low cache hit % (below) = disk-bound reads are the bottleneck; check EBS IOPS / io scheduler latency.*\n\n###### Source:\n[nodestore/backend/NuDBFactory.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/nodestore/backend/NuDBFactory.cpp)\n\n###### Function:\n`NuDB backend read path`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 91 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), 1), \"series\", \"NuDB us/read\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "µs", + "custom": { + "axisLabel": "Latency (µs/read)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 21 + }, + { + "title": "I/O Scheduler Latency p95", + "description": "###### What this is:\n*95th-percentile latency of the node's internal I/O service queue - the async task scheduler that dispatches network and disk callbacks.*\n\n###### How it's computed:\n*histogram_quantile(0.95, rate(ios_latency_milliseconds_bucket[$__rate_interval])).*\n\n###### Reading it:\n*Low and flat when the event loop is responsive. Rising p95 means callbacks are queuing behind long-running work (often blocking disk reads).*\n\n###### Healthy range:\n*< ~10ms p95.*\n\n###### Watch for:\n*p95 climbing into hundreds of ms = the I/O service is saturated; the node cannot service network/disk events promptly, stalling sync.*\n\n###### Source:\n[core/impl/Workers.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/Workers.cpp)\n\n###### Function:\n`io_service latency histogram`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 91 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(ios_latency_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"IO Service p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "p95 Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 22 + }, + { + "title": "NuDB Cache Hit Ratio", + "description": "###### What this is:\n*Fraction of nodestore reads served from cache rather than disk. A cold cache (fresh NuDB after a wipe) forces almost every read to disk - the root cause of high read latency during catch-up.*\n\n###### How it's computed:\n*rate(nodestore_state{metric=\"node_reads_hit\"}[$__rate_interval]) / rate(nodestore_state{metric=\"node_reads_total\"}[$__rate_interval]).*\n\n###### Reading it:\n*Near 1.0 (100%) on a warm node. Near 0 right after a wipe/restart, climbing as the working set warms.*\n\n###### Healthy range:\n*> ~0.9 warm; near 0 briefly after a cold start.*\n\n###### Watch for:\n*Persistently low hit ratio on a node that should be warm = working set does not fit cache, or continuous re-acquisition churning the cache.*\n\n###### Source:\n[nodestore/backend/NuDBFactory.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/nodestore/backend/NuDBFactory.cpp)\n\n###### Function:\n`TaggedCache hit accounting`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 99 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), 1), \"series\", \"NuDB Hit Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percentunit", + "custom": { + "axisLabel": "Hit Ratio", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 23 + }, + { + "title": "NuDB Read Pressure", + "description": "###### What this is:\n*Nodestore read throughput and backlog: reads per second, the pending read queue depth, and how many read threads are active. Shows disk saturation depth.*\n\n###### How it's computed:\n*rate(node_reads_total), and gauges read_queue / read_threads_running from nodestore_state.*\n\n###### Reading it:\n*reads/s spikes during catch-up. A growing read_queue with all read threads busy = disk cannot keep up with demand.*\n\n###### Healthy range:\n*read_queue ~0 and reads/s low on a warm synced node.*\n\n###### Watch for:\n*read_queue climbing while read_threads_running is pinned at read_threads_total = disk-bound; the IOPS ceiling is the limiter.*\n\n###### Source:\n[nodestore/backend/NuDBFactory.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/nodestore/backend/NuDBFactory.cpp)\n\n###### Function:\n`NuDB read scheduler`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 99 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval]), \"series\", \"Reads/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_queue\"}, \"series\", \"Read Queue\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_threads_running\"}, \"series\", \"Read Threads Running\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Reads/s & Depth", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 24 + }, + { + "title": "Job Queue Depth", + "description": "###### What this is:\n*Total number of jobs currently queued across the JobQueue. A backlog means work is arriving faster than worker threads can drain it.*\n\n###### How it's computed:\n*jobq_job_count gauge (total queued jobs), per node.*\n\n###### Reading it:\n*Near 0 when keeping up. Sustained positive depth = the node is overloaded or blocked on a downstream resource (usually disk reads).*\n\n###### Healthy range:\n*~0 on a healthy node.*\n\n###### Watch for:\n*Depth climbing in step with Validated Ledger Age = the queue backlog is why the node is falling behind.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::getJobCountTotal`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 107 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(jobq_job_count{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Job Queue Depth\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Queued Jobs", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 25 + }, + { + "title": "Load Factor & Peers", + "description": "###### What this is:\n*Server load factor (fee/throttle multiplier; 1 = unloaded) alongside active inbound/outbound peer counts. Rules out overload and insufficient fetch sources as causes.*\n\n###### How it's computed:\n*load_factor_metrics{metric=\"load_factor\"}, and peer_finder_active_inbound_peers / _outbound_peers gauges.*\n\n###### Reading it:\n*load_factor at 1 = no local overload. Outbound peers should be healthy (~10+); too few peers limits how fast the node can fetch ledger data.*\n\n###### Healthy range:\n*load_factor = 1; outbound peers >= ~8.*\n\n###### Watch for:\n*load_factor > 1 = local overload throttling; very low peer count = not enough sources to acquire history from (a distinct sync bottleneck).*\n\n###### Source:\n[overlay/detail/OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl load / PeerFinder counts`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 107 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor\"}, \"series\", \"Load Factor\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_finder_active_outbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Outbound Peers\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_finder_active_inbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Inbound Peers\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Factor / Peer Count", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 26 + }, + { + "title": "Job Queue Saturation", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 115 + }, + "collapsed": false, + "panels": [], + "id": 27 + }, + { + "title": "Job Queue Backlog and Deferred by Type", + "description": "###### What this is:\n*Per-job-type queue depth, two series per type. Waiting is the whole backlog: every job enqueued for that type that has not started yet. Deferred is the subset of that backlog that is blocked specifically because the type is already running at its concurrency limit. Deferred is the leading indicator of backpressure, because JobQueue::addJob never rejects a job -- it always returns success and defers instead, so a capped type under pressure produces no error and no dropped work. Without these the only evidence is latency, which appears after the harm is already done.*\n\n###### How it's computed:\n*Two targets, each the top 10 gauges by current value: jobq__waiting and jobq__deferred. Both are live depths published by JobQueue::collect() under the same mutex that guards the counters, so the pair is always read at the same instant and is directly comparable. Gauges exist only for non-special job types, so the 11 special types -- the ones declared with a limit of 0, which bypass the limit logic entirely and therefore never defer -- do not appear on either series.*\n\n###### Reading it:\n*Read the two together; the ratio is the diagnostic, not either value alone. Deferred is always a subset of waiting, because addRefCountedJob increments waiting for every job and deferred only for the ones that arrive while the type is at its limit. Waiting high with deferred at zero means the type has spare slots and the backlog is just arrival burstiness -- it will drain without intervention. Waiting high with deferred also high means the concurrency limit is the binding constraint, not the work. Both near zero is the normal state. These are depths, not rates: the value is how many jobs are queued right now. finishJob drains deferred one per completion, so a deferred line that stays elevated means arrivals are outpacing completions rather than one isolated burst. Only the 10 highest series per state are drawn, which on an idle node is arbitrary among the zeros and under load is exactly the types under pressure.*\n\n###### Healthy range:\n*Deferred zero on all types. Waiting near zero, with brief spikes during ledger close.*\n\n###### Watch for:\n*ledgerrequest deferred above zero: the 3-slot ledgerRequest queue is full, so TMGetLedger service to syncing peers is being delayed. Use LedgerReq Wait by Handler next to see which of its two producers is responsible. ledgerdata or fetchtxndata deferred: inbound ledger data cannot be absorbed fast enough, which is what makes validated ledger age grow. A waiting line that climbs steadily while deferred stays flat points at the worker pool or at slow jobs rather than at the limit. Note both are sampled by the collector hook once per interval, so a sub-second spike can be missed; a reading of zero is not proof that nothing was ever queued or deferred.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::addRefCountedJob / JobQueue::collect`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 116 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\"jobq_.*_deferred\"}), \"series\", \"$1 Deferred\", \"__name__\", \"jobq_(.*)_deferred\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\"jobq_.*_waiting\"}), \"series\", \"$1 Waiting\", \"__name__\", \"jobq_(.*)_waiting\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Queued Jobs (Waiting / Deferred)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 28 + }, + { + "title": "LedgerReq Wait by Handler", + "description": "###### What this is:\n*Queue wait for the ledgerRequest job type, split by which handler enqueued the job. The type has a concurrency limit of 3 and two producers that compete for those slots: RcvGetLedger, which serves TMGetLedger to syncing peers, and RcvGetObjByHash, which serves TMGetObjectByHash. Both report the same job_type, so without the handler split a wait spike cannot be attributed to either.*\n\n###### How it's computed:\n*p99 of job_queued_us for job_type=\"ledgerRequest\", grouped by the handler label. The handler value is the addJob name passed through a sanitiser that keeps letters-only names and folds anything else to \"other\", which bounds the label domain.*\n\n###### Reading it:\n*This is the panel that answers which producer is starving the 3-slot queue. Both lines high together means the queue is genuinely oversubscribed and both kinds of peer request are being delayed. One line high while the other is flat means that producer is arriving faster than 3 concurrent slots can absorb, and it is the one delaying the other. Wait is queue time only, so a high line here is contention, not slow work; the work itself is on the GetObject Handler Latency Breakdown panel.*\n\n###### Healthy range:\n*Single-digit to low-tens of milliseconds p99 for both handlers, matching the wider Job Queue Wait p95 By Type panel.*\n\n###### Watch for:\n*RcvGetObjByHash wait climbing: one in-bounds TMGetObjectByHash request can perform thousands of NodeStore lookups, so a few concurrent ones occupy every slot and delay TMGetLedger to peers that are themselves syncing. Cross-check Job Queue Backlog and Deferred by Type for jobq_ledgerrequest_deferred above zero to confirm the limit, not the work, is the binding constraint.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::processTask -> PerfLog::jobStart`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 116 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, handler, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"ledgerRequest\", handler=~\"$handler\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"$1 q-wait p99\", \"handler\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "µs", + "custom": { + "axisLabel": "p99 Wait (μs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 29 + } + ], + "schemaVersion": 39, + "tags": ["ledger", "sync"], + "templating": { + "list": [ + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "handler", + "label": "Handler", + "description": "Filter by enqueueing handler (sanitised addJob name, e.g. RcvGetLedger)", + "type": "query", + "query": "label_values(job_queued_total, handler)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Ledger Data & Sync", + "uid": "ledger-data-sync" +} diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json new file mode 100644 index 00000000000..a7eeea4ebf0 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -0,0 +1,567 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Ledger Build Rate", + "description": "###### What this is:\n*How many new ledgers this node finishes building per second.*\n\n###### How it's computed:\n*Per-second rate of completed ledger-build operations, averaged over 5 minutes and split by node.*\n\n###### Reading it:\n*A steady flat line; the value should track the network close cadence.*\n\n###### Healthy range:\n*About 0.2-0.3 ledgers/sec on mainnet (roughly one every 3-5s); workload-dependent on test networks.*\n\n###### Watch for:\n*A drop toward zero (node fell out of sync or stalled) or a value well above the network rate (rebuilding history).*\n\n###### Keywords:\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`\n\n###### References:\n[Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-build)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.build\"}[$__rate_interval])), \"series\", \"Builds / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + } + }, + { + "title": "Ledger Build Duration", + "description": "###### What this is:\n*The time taken to build a single ledger, at the 95th percentile.*\n\n###### How it's computed:\n*95th-percentile of ledger-build durations over a 5-minute window, per node.*\n\n###### Reading it:\n*Lower is better; the line should stay well under the ledger interval.*\n\n###### Healthy range:\n*Typically tens to low hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained rises approaching the close interval, which indicate heavy transaction sets or disk/I/O pressure.*\n\n###### Keywords:\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`\n\n###### References:\n[Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-build)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.build\"}[5m]))), \"series\", \"P95 Build Duration\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Ledger Validation Rate", + "description": "###### What this is:\n*How often ledgers reach full validation (accepted by the trusted validator quorum) per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-validation events over 5 minutes, per node.*\n\n###### Reading it:\n*Should closely match the build rate under normal, in-sync operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; workload-dependent elsewhere.*\n\n###### Watch for:\n*A validation rate that lags the build rate, signalling the node is building ahead of the network consensus it trusts.*\n\n###### Keywords:\n- **Ledger validation** *(network event)* \u2014 the second consensus stage where the node confirms a built ledger matches the trusted validator quorum and marks it final.\n- **Validation quorum** *(network-wide)* \u2014 the minimum number of agreeing trusted validations needed to declare a ledger fully validated.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`\n\n###### References:\n[Ledger validation](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Validation quorum](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-validation)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.validate\"}[$__rate_interval])), \"series\", \"Validations / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + } + }, + { + "title": "Ledger Build Duration Heatmap", + "description": "###### What this is:\n*The full distribution of ledger-build times over the window, not just a single percentile.*\n\n###### How it's computed:\n*Counts of ledger builds falling in each duration band per 5-minute window, shown as color density.*\n\n###### Reading it:\n*A tight low band is healthy; scattered high cells mean occasional slow builds.*\n\n###### Healthy range:\n*Most mass concentrated in the low-millisecond bands; workload-dependent.*\n\n###### Watch for:\n*A second cluster of hot cells at high durations (bimodal build times) hidden by percentile charts.*\n\n###### Keywords:\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`\n\n###### References:\n[Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-build)", + "type": "heatmap", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "yAxis": { + "axisLabel": "Duration (ms)" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.build\"}[5m])) by (le)", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + } + }, + { + "title": "Transaction Apply Duration", + "description": "###### What this is:\n*The time spent applying the agreed transaction set into the new ledger, at the 95th percentile.*\n\n###### How it's computed:\n*95th-percentile of transaction-apply durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; this is a large share of total build time.*\n\n###### Healthy range:\n*A few to tens of milliseconds; scales with transaction volume per ledger.*\n\n###### Watch for:\n*Spikes during large or expensive transaction sets, which push out overall ledger build time.*\n\n###### Keywords:\n- **Transaction apply phase** *(per node)* \u2014 the step that executes the agreed transaction set into the new ledger during a close.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`applyTransactions`\n\n###### References:\n[Transaction apply phase](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-apply-phase)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.apply\"}[5m]))), \"series\", \"P95 tx.apply\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Transaction Apply Rate", + "description": "###### What this is:\n*How often the transaction-apply phase runs per second (once per ledger build).*\n\n###### How it's computed:\n*Per-second rate of transaction-apply operations over 5 minutes, per node.*\n\n###### Reading it:\n*Should track the ledger build rate almost exactly.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; workload-dependent.*\n\n###### Watch for:\n*Divergence from the build rate, which would indicate a metric or pipeline anomaly.*\n\n###### Keywords:\n- **Transaction apply phase** *(per node)* \u2014 the step that executes the agreed transaction set into the new ledger during a close.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`applyTransactions`\n\n###### References:\n[Transaction apply phase](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-apply-phase)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.apply\"}[$__rate_interval])), \"series\", \"tx.apply / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Ledger Store Rate", + "description": "###### What this is:\n*How often completed ledgers are written into ledger history per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-store operations over 5 minutes, per node.*\n\n###### Reading it:\n*Should match the build rate during normal operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; can burst higher while backfilling history.*\n\n###### Watch for:\n*A store rate below the build rate (storage falling behind) or a stall at zero.*\n\n###### Keywords:\n- **Ledger store** *(per node)* \u2014 writing a completed ledger into the node's ledger history on disk.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Consensus stall** *(per node)* \u2014 a health check reporting that consensus is not making forward progress.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::storeLedger`\n\n###### References:\n[Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus stall](https://xrpl.org/docs/concepts/consensus-protocol/consensus-principles-and-rules) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-store)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.store\"}[$__rate_interval])), \"series\", \"Stores / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + } + }, + { + "title": "Build vs Close Duration", + "description": "###### What this is:\n*Ledger build time compared with the full consensus round duration, both at the 95th percentile.*\n\n###### How it's computed:\n*Two 95th-percentile duration series over 5 minutes: ledger construction (ledger.build span) and the whole consensus round from open to accept (consensus.round span), per node.*\n\n###### Reading it:\n*Build should sit well below the round; the gap is consensus wait time (proposing, converging, validating) outside construction.*\n\n###### Healthy range:\n*Round tracks the network close interval (~3-5s on mainnet); build is a fraction of it (tens to hundreds of ms).*\n\n###### Watch for:\n*Build time approaching the round duration \u2014 construction is dominating the close and leaving little slack.*\n\n###### Note:\n*The close series uses consensus.round, not consensus.ledger_close: the latter span only wraps the onClose() prologue (sub-millisecond) and is not the ledger close time.*\n\n###### Keywords:\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Consensus mode** *(per node)* \u2014 the node's role/health in the current round: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Ledger close** *(network event)* \u2014 the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp) \u00b7 [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`buildLedgerImpl ; RCLConsensus::Adaptor::onClose (round span)`\n\n###### References:\n[Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-round)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.build\"}[5m]))), \"series\", \"P95 ledger.build\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.round\"}[5m]))), \"series\", \"P95 Close (consensus.round)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Ledger Close Interval & Age", + "description": "###### What this is:\n*The wall-clock time between consecutive ledger closes — the network close cadence.*\n\n###### How it's computed:\n*Close Interval: 1 / rate(ledgers_closed_total), the average seconds between closes from the monotonic close counter (scrape-independent, unlike a gauge delta which would alias to the scrape period). Last-Close Age: time() minus the last-close network time (server_info last_close_time gauge + Ripple-epoch offset), i.e. seconds since the last ledger closed.*\n\n###### Reading it:\n*A steady line near the network's target close interval.*\n\n###### Healthy range:\n*About 3-5s on mainnet; workload-dependent on test networks.*\n\n###### Watch for:\n*A rising interval (consensus slowing or the node lagging) or a flat line at zero (ledgers no longer closing).*\n\n###### Keywords:\n- **Ledger close** *(network event)* — the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n- **Ledger close interval** *(network-wide)* — the network's steady ledger rhythm — roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Consensus** *(network event)* — the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp) · [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`registerServerInfoGauge (last_close_time) ; ledgers_closed_total`\n\n###### References:\n[Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) · [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-close)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(time() - (server_info{metric=\"last_close_time\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} + 946684800), \"series\", \"Last-Close Age (s)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(1 / sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(ledgers_closed_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Close Interval\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "s", + "custom": { + "axisLabel": "Seconds", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["ledger"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Ledger Operations", + "uid": "ledger-operations", + "refresh": "30s", + "description": "What this shows: Ledger construction, validation, and storage activity and timing for this node. \u2014 Use it to: Confirm ledgers are being built, validated, and stored on schedule and find the slow stage when they are not." +} diff --git a/docker/telemetry/grafana/dashboards/network-traffic.json b/docker/telemetry/grafana/dashboards/network-traffic.json new file mode 100644 index 00000000000..a6a98a5e658 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/network-traffic.json @@ -0,0 +1,988 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Peer connectivity and overlay bandwidth for this node: peer counts, disconnects, total bytes and messages exchanged, and the transaction, proposal, and validation traffic that flows across the peer-to-peer network. \u2014 Use it to: Gauge peer health and overall overlay bandwidth and see which message types dominate network traffic.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Active Peers", + "description": "###### What this is:\n*Number of active inbound and outbound peer connections the node currently holds.*\n\n###### How it's computed:\n*Current value of the inbound and outbound active-peer counts per node.*\n\n###### Reading it:\n*Outbound is what the node dials out; inbound is what others open to it. Both should be stable.*\n\n###### Healthy range:\n*roughly 10-21 outbound and 0-85 inbound on mainnet, depending on config.*\n\n###### Watch for:\n*Outbound dropping toward zero (isolation) or inbound pinned at the limit with churn (connection pressure).*\n\n###### Source:\n[PeerfinderManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/PeerfinderManager.cpp)\n\n###### Function:\n`Logic Stats ctor`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_finder_active_inbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Inbound Peers\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_finder_active_outbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Outbound Peers\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Peers", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Peer Disconnects", + "description": "###### What this is:\n*Peer connections that dropped, as a per-interval increase.*\n\n###### How it's computed:\n*Disconnect events per rate-interval, per node.*\n\n###### Reading it:\n*A flat or slowly rising line is normal; the slope matters more than the absolute value.*\n\n###### Healthy range:\n*workload-dependent; slow, steady growth.*\n\n###### Watch for:\n*Sharp step-ups in the slope (network instability, resource exhaustion, or many peers dropping the node at once).*\n\n###### Source:\n[OverlayImpl.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.h)\n\n###### Function:\n`OverlayImpl::Stats ctor`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(increase(overlay_peer_disconnects{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Disconnects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Disconnects per interval", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Total Network Bytes", + "description": "###### What this is:\n*Total bytes received and sent across all peer connections.*\n\n###### How it's computed:\n*Per-second in/out byte rate per node.*\n\n###### Reading it:\n*Overall bandwidth footprint; in and out usually track network activity together.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*Sudden sustained jumps not matched by ledger or transaction activity (relay storms or a noisy peer).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(total_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(total_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Total Network Messages", + "description": "###### What this is:\n*Total messages received and sent across all peer connections.*\n\n###### How it's computed:\n*Per-second in/out message rate per node.*\n\n###### Reading it:\n*Overall message throughput of the overlay; complements the byte totals.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*Message count climbing far faster than bytes (many tiny messages, possible flooding).*\n\n###### Keywords:\n- **Overlay** *(per node)* \u2014 the peer-to-peer network layer over which nodes exchange transactions, proposals, and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Overlay](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(total_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Messages In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(total_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Messages Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Transaction Traffic", + "description": "###### What this is:\n*Transaction relay messages in and out, plus duplicate transaction messages received.*\n\n###### How it's computed:\n*Per-second message rate for the transaction and transaction-duplicate categories.*\n\n###### Reading it:\n*In/out rise with network transaction volume; duplicates are transactions the node already had.*\n\n###### Healthy range:\n*workload-dependent; duplicates a modest fraction of inbound.*\n\n###### Watch for:\n*Duplicate inbound approaching or exceeding unique inbound (redundant relay), or a sharp spike suggesting transaction flooding.*\n\n###### Keywords:\n- **Trusted / untrusted / duplicate** *(per node)* \u2014 message classification: trusted (from UNL validators), untrusted (others), or duplicate (already seen).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Trusted / untrusted / duplicate](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#trusted-untrusted-duplicate)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(transactions_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Messages In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(transactions_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Messages Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(transactions_duplicate_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Duplicate In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Proposal Traffic", + "description": "###### What this is:\n*Consensus proposal messages in/out, plus untrusted and duplicate proposal messages received.*\n\n###### How it's computed:\n*Per-second message rate for the proposal, proposal-untrusted, and proposal-duplicate categories.*\n\n###### Reading it:\n*Trusted in/out track consensus rounds; untrusted come from validators not on this node's trusted list.*\n\n###### Healthy range:\n*workload-dependent; untrusted and duplicates low relative to trusted.*\n\n###### Watch for:\n*High untrusted (trusted-list misconfiguration) or high duplicates (inefficient relay or proposal spam).*\n\n###### Keywords:\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Trusted / untrusted / duplicate** *(per node)* \u2014 message classification: trusted (from UNL validators), untrusted (others), or duplicate (already seen).\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Trusted / untrusted / duplicate](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proposals_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Proposals In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proposals_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Proposals Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proposals_untrusted_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Untrusted In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proposals_duplicate_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Duplicate In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Validation Traffic", + "description": "###### What this is:\n*Validation messages in/out, plus untrusted and duplicate validation messages received.*\n\n###### How it's computed:\n*Per-second message rate for the validation, validation-untrusted, and validation-duplicate categories.*\n\n###### Reading it:\n*Trusted validations should arrive steadily each ledger; untrusted come from non-trusted validators.*\n\n###### Healthy range:\n*workload-dependent; untrusted and duplicates low relative to trusted.*\n\n###### Watch for:\n*Rising untrusted or duplicate validations (trusted-list health issues or validation spam).*\n\n###### Keywords:\n- **Trusted / untrusted / duplicate** *(per node)* \u2014 message classification: trusted (from UNL validators), untrusted (others), or duplicate (already seen).\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Trusted / untrusted / duplicate](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#trusted-untrusted-duplicate)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Validations In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Validations Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_untrusted_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Untrusted In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_duplicate_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Duplicate In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Overlay Traffic by Category (Bytes In)", + "description": "###### What this is:\n*Top overlay traffic categories ranked by inbound bytes, excluding the all-traffic total.*\n\n###### How it's computed:\n*Top categories by inbound byte rate per node.*\n\n###### Reading it:\n*Shows which message types dominate receive bandwidth right now.*\n\n###### Healthy range:\n*workload-dependent; transactions, proposals, and validations typically lead on a synced node.*\n\n###### Watch for:\n*A fetch or ledger-data category topping the list (sync activity) or an unexpected category dominating.*\n\n###### Keywords:\n- **Overlay** *(per node)* \u2014 the peer-to-peer network layer over which nodes exchange transactions, proposals, and validations.\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Overlay](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) \u00b7 [Proposal](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(topk(10, label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(transactions_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"transactions_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(proposals_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"proposals_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(validations_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"validations_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(overhead_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"overhead_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(overhead_overlay_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"overhead_overlay_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ping_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ping_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(status_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"status_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(havetxset_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"havetxset_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledgerdata_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledgerdata_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_account_state_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_account_state_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_set_candidate_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_account_state_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_transaction_set_candidate_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_transaction_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(set_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"set_get_bytes_in\",\"\",\"\")), \"series\", \"$1\", \"__name__\", \"(.*)\")" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "transactions_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Transactions" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "proposals_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Proposals" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "validations_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Validations" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "overhead_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Overhead" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "overhead_overlay_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Overhead Overlay" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ping_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Ping" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "status_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Status" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "getobject_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Get Object" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "havetxset_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Have Tx Set" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledgerdata_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Ledger Data" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Ledger Share" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_get_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Ledger Data Get" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Ledger Data Share" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_account_state_node_get_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Account State Node Get" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_account_state_node_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Account State Node Share" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_transaction_node_get_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Transaction Node Get" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_transaction_node_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Transaction Node Share" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_data_transaction_set_candidate_get_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Tx Set Candidate Get" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_account_state_node_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Account State Node Share (Legacy)" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_transaction_set_candidate_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Tx Set Candidate Share" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ledger_transaction_node_share_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Transaction Node Share (Legacy)" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "set_get_bytes_in" + }, + "properties": [ + { + "id": "displayName", + "value": "Set Get" + } + ] + } + ] + } + }, + { + "title": "Duplicate Traffic (Wasted Bandwidth)", + "description": "###### What this is:\n*Throughput of duplicate transaction, proposal, and validation traffic: messages the node had already seen and discarded.*\n\n###### How it's computed:\n*Per-second rate of the duplicate byte counters for each category, in and out.*\n\n###### Reading it:\n*Lower is better; this is bandwidth spent on redundant relays.*\n\n###### Healthy range:\n*workload-dependent; a small fraction of total traffic.*\n\n###### Watch for:\n*Duplicate rate climbing toward the same order as useful traffic (poor relay topology or redundant flooding).*\n\n###### Keywords:\n- **Trusted / untrusted / duplicate** *(per node)* \u2014 message classification: trusted (from UNL validators), untrusted (others), or duplicate (already seen).\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Trusted / untrusted / duplicate](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Proposal](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#trusted-untrusted-duplicate)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(transactions_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Duplicate In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(transactions_duplicate_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Duplicate Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proposals_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Proposals Duplicate In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proposals_duplicate_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Proposals Duplicate Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Validations Duplicate In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_duplicate_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Validations Duplicate Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Throughput", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "All Traffic Categories (Detail)", + "description": "###### What this is:\n*The busiest overlay categories by inbound byte rate over time, excluding the all-traffic total.*\n\n###### How it's computed:\n*Per-second inbound byte rate for the busiest categories, ranked, excluding the all-traffic total.*\n\n###### Reading it:\n*Time-series companion to the category bar view; shows how the traffic mix shifts over the window.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A category ramping up and staying high, or the mix suddenly changing (sync, spam, or a misbehaving peer).*\n\n###### Keywords:\n- **Overlay** *(per node)* \u2014 the peer-to-peer network layer over which nodes exchange transactions, proposals, and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Overlay](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(15, label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_account_state_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_account_state_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_cas_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_cas_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_cas_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_cas_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_fetch_pack_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_fetch_pack_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_fetch_pack_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_fetch_pack_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_ledger_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_ledger_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_transaction_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_transaction_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_transaction_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_transaction_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_transaction_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_transaction_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(getobject_transactions_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"getobject_transactions_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(have_transactions_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"have_transactions_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_account_state_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_account_state_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_account_state_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_account_state_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_set_candidate_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_data_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_data_transaction_set_candidate_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_transaction_node_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_transaction_node_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_transaction_set_candidate_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(ledger_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"ledger_transaction_set_candidate_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(overhead_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"overhead_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(overhead_cluster_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"overhead_cluster_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(overhead_manifest_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"overhead_manifest_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(overhead_overlay_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"overhead_overlay_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(proof_path_request_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"proof_path_request_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(proof_path_response_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"proof_path_response_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(proposals_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"proposals_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(proposals_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"proposals_duplicate_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(proposals_untrusted_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"proposals_untrusted_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(replay_delta_request_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"replay_delta_request_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(replay_delta_response_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"replay_delta_response_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(requested_transactions_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"requested_transactions_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(set_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"set_get_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(set_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"set_share_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(squelch_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"squelch_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(squelch_ignored_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"squelch_ignored_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(squelch_suppressed_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"squelch_suppressed_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(transactions_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"transactions_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(transactions_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"transactions_duplicate_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(unknown_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"unknown_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(validations_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"validations_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(validations_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"validations_duplicate_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(validations_untrusted_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"validations_untrusted_bytes_in\",\"\",\"\") or label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role)(rate(validator_lists_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])),\"__name__\",\"validator_lists_bytes_in\",\"\",\"\")), \"series\", \"$1\", \"__name__\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Throughput", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["network"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Network Traffic", + "uid": "network-traffic", + "refresh": "30s" +} diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json new file mode 100644 index 00000000000..b6eccb81c88 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -0,0 +1,5422 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Operational health of an XRPL node: sync state, ledger progress, caches, storage, job queue, and network economy. — Use it to: Get an at-a-glance read on whether the node is healthy, synced, and keeping up with the network.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*A single at-a-glance verdict: is this node healthy and doing its job (fully synced and keeping up with the network)?*\n\n###### How it's computed:\n*1 (Healthy) only when server_state == Full AND the validated ledger age is under 30s; otherwise 0 (Not Healthy). Combines server_info{metric=\"server_state\"} and ledgermaster_validated_ledger_age.*\n\n###### Reading it:\n*Green \"Healthy\" = full sync and current. Red \"Not Healthy\" = not full, or lagging the network (catching up, flapping, or stalled).*\n\n###### Healthy range:\n*Healthy (green) in steady state.*\n\n###### Watch for:\n*Any sustained Not Healthy — drill into the Operating Mode and Validated Ledger Age panels below to see whether it is a state or a lag problem.*\n\n###### Keywords:\n- **Full** *(per node)* — the node has the current validated ledger and complete recent history.\n- **Validated ledger age** *(per node)* — seconds since the last freshly validated ledger.\n\n###### Computation boundary:\n*Result: Per node — a boolean AND of two native metrics.*\n*Recorded in xrpld code as native metrics (beast::insight); the collector only forwards them; the Grafana query combines them.*\n\n###### Source:\n[app/misc/NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode / getValidatedLedgerAge`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Not Healthy", + "color": "red", + "index": 0 + }, + "1": { + "text": "Healthy", + "color": "green", + "index": 1 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 129, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace((server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_state\"} == bool 4) * on(service_instance_id) (ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} < bool 30), \"series\", \"Node Health\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "title": "Node Health", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How old the most recently validated ledger is, in seconds.*\n\n###### How it's computed:\n*Current value of the validated-ledger-age gauge, refreshed each collection interval.*\n\n###### Reading it:\n*Lower is better; it should stay near the network close interval.*\n\n###### Healthy range:\n*Under about 7 seconds on a healthy node.*\n\n###### Watch for:\n*Values above 20 seconds mean the node is falling behind the network.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Ledger close interval** *(network-wide)* — the network's steady ledger rhythm — roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/LedgerMaster.h)\n\n###### Function:\n`LedgerMaster::Stats::collectMetrics`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 67, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Validated Age\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Validated Ledger Age", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How old the most recently published ledger is, in seconds.*\n\n###### How it's computed:\n*Current value of the published-ledger-age gauge, refreshed each collection interval.*\n\n###### Reading it:\n*Lower is better; should track close to the validated ledger age.*\n\n###### Healthy range:\n*Under about 7 seconds on a healthy node.*\n\n###### Watch for:\n*A growing gap above the validated age indicates the publish pipeline is backing up.*\n\n###### Keywords:\n- **Published ledger** *(per node)* — the most recent validated ledger the node has finished publishing to its subscribers.\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/LedgerMaster.h)\n\n###### Function:\n`LedgerMaster::Stats::collectMetrics`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#published-ledger)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 68, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledgermaster_published_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Published Age\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Published Ledger Age", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How often the node requests a ledger from its peers.*\n\n###### How it's computed:\n*Per-second rate of ledger-fetch requests over a 5-minute window.*\n\n###### Reading it:\n*Near zero in steady state; elevated while catching up.*\n\n###### Healthy range:\n*Close to zero once fully synced.*\n\n###### Watch for:\n*A sustained high rate means the node is repeatedly missing ledgers and back-filling from peers.*\n\n###### Keywords:\n- **Ledger acquire (inbound fetch)** *(per node)* — fetching a specific missing ledger from peers; tracked by duration and outcome (complete/failed).\n- **Back-fill / catch-up** *(per node)* — fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedgers.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedgers.cpp)\n\n###### Function:\n`InboundLedgersImp`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-acquire-inbound-fetch)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 69, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_fetches_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetches / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger Fetch Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate at which a locally built ledger hash fails to match the network-validated hash.*\n\n###### How it's computed:\n*Per-second rate of history-mismatch events over a 5-minute window.*\n\n###### Reading it:\n*Flat at zero is the only healthy reading.*\n\n###### Healthy range:\n*Zero.*\n\n###### Watch for:\n*Any nonzero value indicates consensus divergence or database corruption and warrants immediate investigation.*\n\n###### Keywords:\n- **Ledger history mismatch** *(per node)* — when a locally built ledger's hash does not match the network-validated hash.\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Consensus** *(network event)* — the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementLedgerHistoryMismatch`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-history-mismatch)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0.01 + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 70, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledger_history_mismatch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Mismatches / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger History Mismatches", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The lag between published and validated ledger ages, in seconds.*\n\n###### How it's computed:\n*Published ledger age minus validated ledger age, as a single derived value.*\n\n###### Reading it:\n*Near zero is healthy; a positive value is how far publishing trails validation.*\n\n###### Healthy range:\n*Close to zero.*\n\n###### Watch for:\n*A growing gap means the publish pipeline is falling behind and subscribers may see stale data.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Published ledger** *(per node)* — the most recent validated ledger the node has finished publishing to its subscribers.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/LedgerMaster.h)\n\n###### Function:\n`LedgerMaster::Stats::collectMetrics`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 71, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledgermaster_published_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} - ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Publish Gap\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger Publish Gap", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Latency of the I/O service loop, at the 95th percentile in milliseconds.*\n\n###### How it's computed:\n*I/O loop latency samples aggregated to their 95th percentile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; it reflects how promptly queued I/O work runs.*\n\n###### Healthy range:\n*A few milliseconds; brief spikes are tolerable.*\n\n###### Watch for:\n*Sustained values above a few hundred ms indicate thread-pool saturation or blocking operations.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 72, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(ios_latency_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 I/O Latency\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "I/O Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 121, + "title": "Operating Mode", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Fraction of recent wall-clock time the node spent in each operating mode (Disconnected, Connected, Syncing, Tracking, Full).*\n\n###### How it's computed:\n*Per-second rate of each per-mode duration counter divided by the sum of all five mode rates, giving each mode's time share.*\n\n###### Reading it:\n*The Full share should sit at or near 1.0 and dominate; other shares should be near 0.*\n\n###### Healthy range:\n*Full share close to 1.0.*\n\n###### Watch for:\n*Share accumulating in Syncing, Connected, or Disconnected means the node is not staying fully synced.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time Share", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 74, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Tracking\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Syncing\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Connected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Disconnected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "E" + } + ], + "title": "Operating Mode (Time Share)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Cumulative count of transitions into each operating mode.*\n\n###### How it's computed:\n*Current value of each per-mode transition counter, plotted as lines.*\n\n###### Reading it:\n*Flat lines are healthy; steps up mean the node changed mode.*\n\n###### Healthy range:\n*Few transitions once the node is stable in Full mode.*\n\n###### Watch for:\n*Frequent transitions out of Full, or into Disconnected or Syncing, indicate instability.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Transitions", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 75, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_accounting_full_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_accounting_tracking_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Tracking\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_accounting_syncing_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Syncing\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_accounting_connected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Connected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_accounting_disconnected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Disconnected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "E" + } + ], + "title": "Operating Mode Transitions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Which operating mode the node is accumulating time in right now, one line per state, normalized to seconds per second.*\n\n###### How it's computed:\n*Per-second rate of each state's duration counter, scaled from microseconds to seconds. The five lines sum to about 1.0 because the node is always in exactly one state.*\n\n###### Reading it:\n*The line sitting near 1.0 is the state the node is currently in; the others sit at 0. A handover between two lines marks a state change, and its width is how long that state lasted.*\n\n###### Healthy range:\n*Full near 1.0 with every other line at 0.*\n\n###### Watch for:\n*Time accumulating in Connected or Syncing means the node is catching up rather than serving; repeated handovers mean it is flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rate (s/s)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 76, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / 1000000, \"series\", \"Full Mode Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / 1000000, \"series\", \"Tracking Mode Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / 1000000, \"series\", \"Syncing Mode Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / 1000000, \"series\", \"Connected Mode Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / 1000000, \"series\", \"Disconnected Mode Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "E" + } + ], + "title": "State Duration Rate (All States)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The share of recent wall-clock time the node spent in Full mode.*\n\n###### How it's computed:\n*Per-second rate of the Full-mode duration counter divided by the sum of the per-second rates of all five mode duration counters.*\n\n###### Reading it:\n*Higher is better; 1.0 means the node was fully synced for the entire window.*\n\n###### Healthy range:\n*At or above 0.99.*\n\n###### Watch for:\n*Values dropping below 0.9, meaning the node spent meaningful time outside Full mode.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 0.9 + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 77, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"% Time in Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "% Time in Full", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The node's operating mode over time as a colored timeline (Disconnected, Connected, Syncing, Tracking, Full).*\n\n###### How it's computed:\n*Current value of the server-state gauge mapped to a mode name; equal consecutive samples are merged into a single band.*\n\n###### Reading it:\n*A solid green Full band across the window is the goal; other colors mark periods the node was not fully synced.*\n\n###### Healthy range:\n*Continuously Full (green).*\n\n###### Watch for:\n*Bands of Syncing, Connected, or Disconnected, which pinpoint exactly when the node dropped out of Full.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 80, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "Disconnected" + } + }, + "type": "value" + }, + { + "options": { + "1": { + "color": "orange", + "text": "Connected" + } + }, + "type": "value" + }, + { + "options": { + "2": { + "color": "yellow", + "text": "Syncing" + } + }, + "type": "value" + }, + { + "options": { + "3": { + "color": "blue", + "text": "Tracking" + } + }, + "type": "value" + }, + { + "options": { + "4": { + "color": "green", + "text": "Full" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 24, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 78, + "options": { + "alignValue": "left", + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_state\"}", + "legendFormat": "State [{{service_instance_id}}]", + "refId": "A" + } + ], + "title": "Operating Mode (State Timeline)", + "type": "state-timeline" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 89 + }, + "id": 122, + "title": "NodeStore I/O", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Keywords:\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Operations / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 80, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval]), \"series\", \"Reads Total\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]), \"series\", \"Reads Hit (cache)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_writes\"}[$__rate_interval]), \"series\", \"Writes Total\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + } + ], + "title": "NodeStore Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Keywords:\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Operations / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 90 + }, + "id": 81, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]) / ignoring(metric) rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval]), \"series\", \"Cache Hit Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "NodeStore Cache Hit Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Instantaneous write-load score and read-queue depth of the object store.*\n\n###### How it's computed:\n*Current values of the write-load and read-queue gauges, plotted over time.*\n\n###### Reading it:\n*Lower is better for both; short, flat lines are healthy.*\n\n###### Healthy range:\n*Write load near zero and read queue in low double digits or less.*\n\n###### Watch for:\n*High write load means back-end pressure; a high read queue means the prefetch threads are saturated.*\n\n###### Keywords:\n- **Transaction queue (TxQ)** *(per node)* — holds transactions that meet local cost but not the open-ledger cost, to include in a later ledger.\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n- **Read threads / read queue / write load** *(per node)* — NodeStore back-end I/O internals — worker threads reading, their queue depth, and write pressure.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Transaction queue (TxQ)](https://xrpl.org/docs/concepts/transactions/transaction-queue) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-queue-txq)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 102 + }, + "id": 82, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"write_load\"}, \"series\", \"Write Load\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_queue\"}, \"series\", \"Read Queue\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "NodeStore Write Load & Read Queue", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Cumulative bytes read from and written to the object-store back end.*\n\n###### How it's computed:\n*Current values of the bytes-read and bytes-written counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope shows throughput.*\n\n###### Healthy range:\n*Smooth growth consistent with ledger and query activity.*\n\n###### Watch for:\n*A sharp acceleration indicates a heavy I/O phase such as sync, replay, or large queries.*\n\n###### Keywords:\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Throughput", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 102 + }, + "id": 83, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_read_bytes\"}[$__rate_interval]), \"series\", \"Bytes Read\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_written_bytes\"}[$__rate_interval]), \"series\", \"Bytes Written\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "NodeStore Bytes Read/Written", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Keywords:\n- **Read threads / read queue / write load** *(per node)* — NodeStore back-end I/O internals — worker threads reading, their queue depth, and write pressure.\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#read-threads-read-queue-write-load)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Threads / Bundle", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 114 + }, + "id": 84, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_threads_running\"}, \"series\", \"Read Threads Running\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_threads_total\"}, \"series\", \"Read Threads Total\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_request_bundle\"}, \"series\", \"Read Request Bundle\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + } + ], + "title": "NodeStore Read Threads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Keywords:\n- **Read threads / read queue / write load** *(per node)* — NodeStore back-end I/O internals — worker threads reading, their queue depth, and write pressure.\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#read-threads-read-queue-write-load)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Threads / Bundle", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 114 + }, + "id": 85, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval]) / 1e6, \"series\", \"Read Busy Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "NodeStore Read Busy Ratio", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 126 + }, + "id": 123, + "title": "Jobs", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of jobs currently waiting in the internal work queue.*\n\n###### How it's computed:\n*Current value of the job-count gauge, plotted over time.*\n\n###### Reading it:\n*Lower is better; a near-flat low line means the node keeps up with its workload.*\n\n###### Healthy range:\n*Low single digits at idle, brief bumps under load.*\n\n###### Watch for:\n*A sustained high depth means the node cannot process work fast enough, common during replay, heavy RPC, or a request flood.*\n\n###### Source:\n[JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Jobs", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 127 + }, + "id": 87, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(jobq_job_count{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Job Queue Depth\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Job Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Execution time for the most critical job types at the selected quantile.*\n\n###### How it's computed:\n*Per-job-type execution durations aggregated to the chosen quantile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; each line is one job type's run time.*\n\n###### Healthy range:\n*A few to tens of milliseconds for most jobs; workload-dependent.*\n\n###### Watch for:\n*Spikes on key jobs (accept, transaction, write) indicate processing bottlenecks.*\n\n###### Source:\n[JobTypeData.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/core/JobTypeData.h)\n\n###### Function:\n`JobTypeData`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 127 + }, + "id": 88, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"acceptLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Accept Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"advanceLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Advance Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"transaction\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Transaction\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"writeObjects\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Write Objects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"heartbeat\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Heartbeat\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"sweep\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Sweep\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"trustedValidation\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Trusted Validation\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"trustedProposal\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Trusted Proposal\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"publishNewLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Publish New Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"clientRPC\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Client RPC\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{job_type=\"ledgerData\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Ledger Data\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "K" + } + ], + "title": "Key Jobs Execution Time (q$quantile)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Time critical jobs wait in the queue before running, at the selected quantile.*\n\n###### How it's computed:\n*Per-job-type queue-wait durations aggregated to the chosen quantile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; each line is one job type's wait time.*\n\n###### Healthy range:\n*Near zero to a few milliseconds when the queue is keeping up.*\n\n###### Watch for:\n*High waits mean the queue is backlogged and jobs are scheduled late.*\n\n###### Source:\n[JobTypeData.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/core/JobTypeData.h)\n\n###### Function:\n`JobTypeData`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Wait Time (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 139 + }, + "id": 89, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"acceptLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Accept Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"advanceLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Advance Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"transaction\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Transaction\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"writeObjects\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Write Objects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"heartbeat\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Heartbeat\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"sweep\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Sweep\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"trustedValidation\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Trusted Validation\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"trustedProposal\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Trusted Proposal\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"publishNewLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Publish New Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"clientRPC\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Client RPC\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"ledgerData\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"Ledger Data\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "K" + } + ], + "title": "Key Jobs Dequeue Wait Time (q$quantile)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Execution time for every non-special job type at the selected quantile.*\n\n###### How it's computed:\n*Per-job-type execution durations aggregated to the chosen quantile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; this is the full picture behind the Key Jobs view.*\n\n###### Healthy range:\n*A few to tens of milliseconds for most jobs.*\n\n###### Watch for:\n*Broad elevation across many job types indicates systemic processing load.*\n\n###### Source:\n[JobTypeData.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/core/JobTypeData.h)\n\n###### Function:\n`JobTypeData`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 151 + }, + "id": 90, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (service_instance_id, le, job_type, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"$1\", \"job_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "All Jobs Execution Time (Detail)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Queue wait time for every non-special job type at the selected quantile.*\n\n###### How it's computed:\n*Per-job-type queue-wait durations aggregated to the chosen quantile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; this is the full picture behind the Key Jobs wait view.*\n\n###### Healthy range:\n*Near zero to a few milliseconds when the queue keeps up.*\n\n###### Watch for:\n*High waits across many job types indicate systemic queue congestion.*\n\n###### Source:\n[JobTypeData.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/core/JobTypeData.h)\n\n###### Function:\n`JobTypeData`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Wait Time (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 167 + }, + "id": 91, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile($quantile, sum by (service_instance_id, le, job_type, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"$1\", \"job_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "All Jobs Dequeue Wait (Detail)", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 183 + }, + "id": 124, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Hit rates for the SLE, Ledger, and AcceptedLedger caches, from 0 to 1.*\n\n###### How it's computed:\n*Current values of the per-cache hit-rate gauges, plotted as lines.*\n\n###### Reading it:\n*Higher is better; each line is the fraction of lookups served from cache.*\n\n###### Healthy range:\n*Above roughly 0.8 in steady state.*\n\n###### Watch for:\n*Low or falling hit rates indicate cache thrashing and extra back-end reads.*\n\n###### Keywords:\n- **Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger)** *(per node)* — in-memory caches that avoid re-reading or re-computing ledger data; higher hit rates mean less back-end work.\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCacheHitRateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#caches)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hit Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 180 + }, + "id": 93, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"SLE_hit_rate\"}, \"series\", \"SLE Hit Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"ledger_hit_rate\"}, \"series\", \"Ledger Hit Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"AL_hit_rate\"}, \"series\", \"AcceptedLedger Hit Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + } + ], + "title": "Cache Hit Rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Entry counts for the TreeNode cache and track set, the FullBelow cache, and the AcceptedLedger cache.*\n\n###### How it's computed:\n*Current values of the per-cache size gauges, plotted as lines.*\n\n###### Reading it:\n*Stable lines are normal; sizes grow with working set and shrink after sweeps.*\n\n###### Healthy range:\n*Stable within configured limits.*\n\n###### Watch for:\n*Unbounded growth suggests memory pressure or a cache not being swept.*\n\n###### Keywords:\n- **Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger)** *(per node)* — in-memory caches that avoid re-reading or re-computing ledger data; higher hit rates mean less back-end work.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCacheHitRateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#caches)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Entries", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 180 + }, + "id": 94, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"treenode_cache_size\"}, \"series\", \"TreeNode Cache\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"treenode_track_size\"}, \"series\", \"TreeNode Track\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"fullbelow_size\"}, \"series\", \"FullBelow\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(cache_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"AL_size\"}, \"series\", \"AcceptedLedger Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "D" + } + ], + "title": "Cache Sizes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of entries in the FullBelowCache, which tracks subtrees known to be fully present locally.*\n\n###### How it's computed:\n*Current value of the cache size gauge, plotted over time.*\n\n###### Reading it:\n*A stable size is normal; it grows during acquisition and is trimmed by sweeps.*\n\n###### Healthy range:\n*Stable within its configured bound.*\n\n###### Watch for:\n*Unbounded growth suggests the cache is not being swept.*\n\n###### Keywords:\n- **Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger)** *(per node)* — in-memory caches that avoid re-reading or re-computing ledger data; higher hit rates mean less back-end work.\n- **Ledger acquire (inbound fetch)** *(per node)* — fetching a specific missing ledger from peers; tracked by duration and outcome (complete/failed).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[TaggedCache.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/basics/TaggedCache.h)\n\n###### Function:\n`TaggedCache::Stats`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#caches)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Entries", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 192 + }, + "id": 95, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(node_family_full_below_cache_size{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"FullBelowCache Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "FullBelowCache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Hit-rate percentage for the FullBelowCache.*\n\n###### How it's computed:\n*Current value of the cache hit-rate gauge.*\n\n###### Reading it:\n*Higher is better; it shows how often cached subtree knowledge is reused.*\n\n###### Healthy range:\n*Above roughly 50 percent in steady state.*\n\n###### Watch for:\n*A low hit rate during steady state means redundant subtree work and warrants investigation.*\n\n###### Keywords:\n- **Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger)** *(per node)* — in-memory caches that avoid re-reading or re-computing ledger data; higher hit rates mean less back-end work.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[TaggedCache.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/basics/TaggedCache.h)\n\n###### Function:\n`TaggedCache::Stats`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#caches)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 25 + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 192 + }, + "id": 96, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": true + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": true, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(node_family_full_below_cache_hit_rate{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Hit Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "FullBelowCache Hit Rate", + "type": "gauge" + } + ], + "title": "Caches", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 184 + }, + "id": 125, + "title": "Server Info", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current state of each XRPL node. Green = FULL (healthy); orange/yellow = syncing in progress; red = disconnected.\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "dark-red", + "index": 0, + "text": "DISCONNECTED" + }, + "1": { + "color": "yellow", + "index": 1, + "text": "CONNECTED" + }, + "2": { + "color": "orange", + "index": 2, + "text": "SYNCING" + }, + "3": { + "color": "light-blue", + "index": 3, + "text": "TRACKING" + }, + "4": { + "color": "green", + "index": 4, + "text": "FULL" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "DISCONNECTED" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CONNECTED" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SYNCING" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "TRACKING" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "FULL" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 185 + }, + "id": 98, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "app": "grafana-assistant-app", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_state\"}", + "instant": false, + "legendFormat": "{{service_instance_id}} [{{xrpl_branch}}]", + "queryType": "range", + "range": true, + "refId": "A" + } + ], + "title": "Server State", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How long the server process has been running, in seconds.*\n\n###### How it's computed:\n*Current value of the uptime gauge.*\n\n###### Reading it:\n*Higher is generally better; a reset to a small value means the process restarted.*\n\n###### Healthy range:\n*Continuously increasing.*\n\n###### Watch for:\n*An unexpected drop to near zero indicates a restart or crash.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "dthms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 185 + }, + "id": 99, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"uptime\"}", + "instant": true, + "legendFormat": "{{service_instance_id}}", + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Total connected peers, inbound plus outbound.*\n\n###### How it's computed:\n*Current value of the peer-count gauge.*\n\n###### Reading it:\n*A stable count in the healthy range is good; too few limits connectivity.*\n\n###### Healthy range:\n*Workload- and config-dependent, typically 10 or more.*\n\n###### Watch for:\n*A sudden drop points to network or connectivity problems; an unusually high inbound count can indicate connection-flood pressure.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "green", + "value": 10 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 197 + }, + "id": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"peers\"}", + "instant": true, + "legendFormat": "{{service_instance_id}}", + "refId": "A" + } + ], + "title": "Peer Count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Sequence number of the current open ledger.*\n\n###### How it's computed:\n*Current value of the open-ledger index gauge.*\n\n###### Reading it:\n*Should climb steadily; the gap above the validated sequence is the ledgers in flight.*\n\n###### Healthy range:\n*One or two ahead of the validated sequence.*\n\n###### Watch for:\n*A large or growing gap above the validated sequence means validation is lagging behind ledger creation.*\n\n###### Keywords:\n- **Ledger index** *(network-wide)* — the sequence number identifying a ledger version; increases by one each close.\n- **Open ledger** *(per node)* — the temporary workspace ledger where incoming transactions are provisionally applied before a close.\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Ledger index](https://xrpl.org/docs/references/protocol/data-types/basic-data-types#ledger-index) · [Open ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-index)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 197 + }, + "id": 103, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"ledger_current_index\"}, \"series\", \"Current Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Current Ledger Index", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The spread in validated ledger sequence across all selected nodes.*\n\n###### How it's computed:\n*Highest validated ledger sequence minus the lowest, across the selected nodes.*\n\n###### Reading it:\n*0 means every node agrees on the same validated ledger; larger means they diverge.*\n\n###### Healthy range:\n*0 to 1 ledger in steady state.*\n\n###### Watch for:\n*A sustained spread above a few ledgers means some nodes are lagging or the fleet is diverging.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Across the selected nodes — the query aggregates instances into one series.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "service_instance_id" + }, + "properties": [ + { + "id": "displayName", + "value": "Node" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Seq" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 209 + }, + "id": 58, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "app": "grafana-assistant-app", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"}) - min(server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"})", + "instant": true, + "legendFormat": "Spread", + "queryType": "instant", + "range": false, + "refId": "A" + } + ], + "title": "Validated Ledger Seq — Convergence (Max − Min)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How far each node's validated ledger lags behind the network tip, in ledgers.*\n\n###### How it's computed:\n*Highest validated ledger sequence across the selected nodes, minus each node's own sequence.*\n\n###### Reading it:\n*0 means the node is at the tip; larger values mean it trails further behind.*\n\n###### Healthy range:\n*0 to 1 ledger on a synced node.*\n\n###### Watch for:\n*A node stuck at a growing value is falling behind and not keeping up with consensus.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Consensus** *(network event)* — the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Across the selected nodes — the query aggregates instances into one series.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 209 + }, + "id": 56, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "app": "grafana-assistant-app", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sort_desc(scalar(max(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})) - server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})", + "instant": true, + "legendFormat": "{{service_instance_id}}", + "queryType": "instant", + "range": false, + "refId": "A" + } + ], + "title": "Validated Ledger Seq — Current (Stat)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The running server's build version string.*\n\n###### How it's computed:\n*Read from the version label of the build-info metric (its value is always 1).*\n\n###### Reading it:\n*Confirms which version each node is running.*\n\n###### Healthy range:\n*The expected release version across all nodes.*\n\n###### Watch for:\n*A node on an unexpected or mismatched version in a fleet.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerBuildInfoGauge`", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 216 + }, + "id": 102, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "app": "grafana-assistant-app", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count by (version) (build_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"})", + "instant": false, + "legendFormat": "{{version}}", + "queryType": "range", + "range": true, + "refId": "A" + } + ], + "title": "Build Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Keywords:\n- **Proposers** *(network event)* — the count of validators whose proposals this node heard in the last closed round.\n- **Convergence time** *(network event)* — the wall-clock time the network took to agree a ledger in a round.\n- **Consensus round** *(network event)* — one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Consensus** *(network event)* — the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Proposers](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#proposers)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "ms", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 216 + }, + "id": 105, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_converge_time_ms\"}, \"series\", \"Converge Time ms\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Last Close — Converge Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Keywords:\n- **Proposers** *(network event)* — the count of validators whose proposals this node heard in the last closed round.\n- **Convergence time** *(network event)* — the wall-clock time the network took to agree a ledger in a round.\n- **Consensus round** *(network event)* — one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Consensus** *(network event)* — the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Proposers](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#proposers)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Proposers", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 228 + }, + "id": 104, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_proposers\"}, \"series\", \"Proposers\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Last Close — Proposers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The wall-clock time between consecutive ledger closes — the network close cadence.*\n\n###### How it's computed:\n*Close Interval: 1 / rate(ledgers_closed_total), the average seconds between closes from the monotonic close counter (scrape-independent, unlike a gauge delta which would alias to the scrape period). Last-Close Age: time() minus the last-close network time (server_info last_close_time gauge + Ripple-epoch offset), i.e. seconds since the last ledger closed.*\n\n###### Reading it:\n*A steady line near the network's target close interval.*\n\n###### Healthy range:\n*About 3-5s on mainnet; workload-dependent on test networks.*\n\n###### Watch for:\n*A rising interval (consensus slowing or the node lagging) or a flat line at zero (ledgers no longer closing).*\n\n###### Keywords:\n- **Ledger close** *(network event)* — the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n- **Ledger close interval** *(network-wide)* — the network's steady ledger rhythm — roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Consensus** *(network event)* — the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp) · [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`registerServerInfoGauge (last_close_time) ; ledgers_closed_total`\n\n###### References:\n[Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) · [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-close)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 240 + }, + "id": 128, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(time() - (server_info{metric=\"last_close_time\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} + 946684800), \"series\", \"Last-Close Age (s)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(1 / sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(ledgers_closed_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Close Interval\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Ledger Close Interval & Age", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*On-disk size of the NuDB object-store back end, in bytes.*\n\n###### How it's computed:\n*Current value of the NuDB storage-size gauge, plotted over time.*\n\n###### Reading it:\n*A smoothly growing line is normal; the slope is the data growth rate.*\n\n###### Healthy range:\n*Gradual growth consistent with retained ledger history.*\n\n###### Watch for:\n*A sudden jump in growth rate can indicate runaway storage or an unexpected back-fill.*\n\n###### Keywords:\n- **NuDB** *(per node)* — the append-only key-value database used as the default NodeStore backend.\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n- **Back-fill / catch-up** *(per node)* — fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStorageDetailGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nudb)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Size", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "decbytes" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 228 + }, + "id": 106, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(storage_detail{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"nudb_bytes\"}, \"series\", \"NuDB Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "NuDB Storage Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Live instance counts for the busiest internal object types.*\n\n###### How it's computed:\n*Current per-type instance counts, showing the top 15 types over time.*\n\n###### Reading it:\n*Stable lines are healthy; each line is one object type's live count.*\n\n###### Healthy range:\n*Steady counts that rise and fall with load.*\n\n###### Watch for:\n*A single type climbing without bound suggests memory pressure or a leak.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerObjectCountGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Instances", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 240 + }, + "id": 107, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": ["last", "max"], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(15, object_count{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", type=~\"$type\"}), \"series\", \"$1\", \"type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Object Instance Counts", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 256 + }, + "id": 126, + "title": "Complete Ledgers & DB", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The contiguous ranges of ledgers the node holds locally.*\n\n###### How it's computed:\n*Current start and end bounds of each complete range, listed as table rows.*\n\n###### Reading it:\n*Fewer ranges is better; one continuous range means an unbroken history.*\n\n###### Healthy range:\n*A single range covering the configured retention window.*\n\n###### Watch for:\n*Many fragmented ranges indicate gaps in stored history from missed or failed fetches.*\n\n###### Keywords:\n- **Complete ledger ranges** *(per node)* — the contiguous spans of ledgers the node holds locally; one unbroken range is ideal.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCompleteLedgersGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#complete-ledger-ranges)", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 257 + }, + "id": 109, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "complete_ledgers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}", + "format": "table", + "instant": true, + "legendFormat": "{{bound}} [range {{index}}] [{{service_instance_id}}]", + "refId": "A" + } + ], + "title": "Complete Ledger Ranges", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Sizes of the relational databases in KB (total, ledger, transaction).*\n\n###### How it's computed:\n*Current values of the per-database size gauges, plotted as lines.*\n\n###### Reading it:\n*Smoothly growing lines are normal; the split shows where storage is used.*\n\n###### Healthy range:\n*Gradual growth consistent with retained history.*\n\n###### Watch for:\n*An abrupt change in growth rate can indicate storage pressure or a pruning issue.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerDbMetricsGauge`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Size (KB)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "deckbytes" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 257 + }, + "id": 110, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(db_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"db_kb_total\"}, \"series\", \"Total KB\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(db_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"db_kb_ledger\"}, \"series\", \"Ledger KB\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(db_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"db_kb_transaction\"}, \"series\", \"Transaction KB\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + } + ], + "title": "Database Sizes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How many historical ledgers the node is back-filling per minute.*\n\n###### How it's computed:\n*Current value of the historical-fetch-per-minute gauge.*\n\n###### Reading it:\n*Near zero once history is complete; elevated while back-filling.*\n\n###### Healthy range:\n*Close to zero in steady state.*\n\n###### Watch for:\n*A sustained high rate means the node is still filling gaps in its stored history.*\n\n###### Keywords:\n- **Historical fetch rate** *(per node)* — how many historical ledgers the node is back-filling per minute.\n- **Back-fill / catch-up** *(per node)* — fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Complete ledger ranges** *(per node)* — the contiguous spans of ledgers the node holds locally; one unbroken range is ideal.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerDbMetricsGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#historical-fetch-rate)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 269 + }, + "id": 111, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(db_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"historical_perminute\"}, \"series\", \"Fetches/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Historical Fetch Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Cumulative count of peers disconnected for exceeding resource limits.*\n\n###### How it's computed:\n*Current value of the resource-disconnect gauge, plotted over time.*\n\n###### Reading it:\n*A flat line is healthy; steps up mean peers were dropped for overuse.*\n\n###### Healthy range:\n*Flat or very slowly rising.*\n\n###### Watch for:\n*A rising line indicates peers are being throttled off, consistent with abusive or misbehaving peers.*\n\n###### Keywords:\n- **Resource disconnect** *(per node)* — a peer dropped for exceeding resource/load limits — the node shedding abusive or overactive peers.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 269 + }, + "id": 112, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"peer_disconnects_resources\"}, \"series\", \"Resource Disconnects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Peer Disconnects (Resources)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 281 + }, + "id": 127, + "title": "Ledger Economy", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The node's local load-based fee factor that scales its minimum transaction cost; baseline 256 at idle.*\n\n###### How it's computed:\n*Current value of the local load-fee economy gauge.*\n\n###### Reading it:\n*Steady at the baseline (256) is normal; higher values mean the node is raising its fee in response to load.*\n\n###### Healthy range:\n*Around 256 (the normal baseline) when idle.*\n\n###### Watch for:\n*A climbing factor, which indicates the node is under transaction load pressure.*\n\n###### Keywords:\n- **Load factor** *(per node)* — a unitless multiplier (1.0 = no load) that scales the base transaction cost as the node comes under load.\n- **Base fee** *(network-wide)* — the baseline transaction cost for a reference transaction under minimum load, in drops.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[Load factor](https://xrpl.org/docs/concepts/transactions/transaction-cost#local-load-cost) · [Base fee](https://xrpl.org/docs/concepts/transactions/transaction-cost) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#load-factor)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "si:drops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 282 + }, + "id": 114, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_economy{metric=\"base_fee_xrp\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Base Fee\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Base Fee (drops)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The minimum XRP balance required to keep an account on the ledger, in drops.*\n\n###### How it's computed:\n*Current value of the reserve-base economy gauge.*\n\n###### Reading it:\n*A stable value is expected; it changes only via network amendment or vote.*\n\n###### Healthy range:\n*The configured network reserve base.*\n\n###### Watch for:\n*An unexpected change outside a known amendment or fee vote.*\n\n###### Keywords:\n- **drops** *(network-wide)* — the smallest XRP unit — one drop is 0.000001 XRP (one millionth).\n- **Reserve (base & owner)** *(network-wide)* — the minimum XRP an account must hold — a base reserve plus an increment per owned ledger object.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[drops](https://xrpl.org/docs/references/protocol/data-types/basic-data-types#specifying-currency-amounts) · [Reserve (base & owner)](https://xrpl.org/docs/concepts/accounts/reserves) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#drops)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "si:drops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 282 + }, + "id": 115, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_economy{metric=\"reserve_base_xrp\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Reserve Base\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Reserve Base (drops)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The additional XRP reserve required per owned ledger object, in drops.*\n\n###### How it's computed:\n*Current value of the reserve-increment economy gauge.*\n\n###### Reading it:\n*A stable value is expected; it changes only via network amendment or vote.*\n\n###### Healthy range:\n*The configured network owner reserve increment.*\n\n###### Watch for:\n*An unexpected change outside a known amendment or fee vote.*\n\n###### Keywords:\n- **Reserve (base & owner)** *(network-wide)* — the minimum XRP an account must hold — a base reserve plus an increment per owned ledger object.\n- **drops** *(network-wide)* — the smallest XRP unit — one drop is 0.000001 XRP (one millionth).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[Reserve (base & owner)](https://xrpl.org/docs/concepts/accounts/reserves) · [drops](https://xrpl.org/docs/references/protocol/data-types/basic-data-types#specifying-currency-amounts) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#reserve-base-owner)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "si:drops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 294 + }, + "id": 116, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_economy{metric=\"reserve_inc_xrp\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Reserve Inc\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Reserve Inc (drops)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Seconds since the last validated ledger closed, plotted over time.*\n\n###### How it's computed:\n*Current value of the ledger-age economy gauge, sampled each interval.*\n\n###### Reading it:\n*Lower is better; it should stay near the ledger close interval. Mirrors the Validated Ledger Age panel.*\n\n###### Healthy range:\n*Under about 10 seconds.*\n\n###### Watch for:\n*Growth beyond the expected close interval, meaning the node is not keeping up with validated ledgers.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Ledger close interval** *(network-wide)* — the network's steady ledger rhythm — roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Ledger Age (s)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 12, + "y": 294 + }, + "id": 117, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_economy{metric=\"ledger_age_seconds\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Ledger Age\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger Age", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The network transaction throughput reported by the ledger economy metrics.*\n\n###### How it's computed:\n*Current value of the transaction-rate economy gauge, plotted over time.*\n\n###### Reading it:\n*Reflects how many transactions are being processed; higher means busier.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A sudden sustained surge can indicate a transaction flood; a drop to zero can indicate the node stopped processing.*\n\n###### Keywords:\n- **Transaction type** *(network-wide)* — the kind of transaction (Payment, OfferCreate, TrustSet, AMM*, NFToken*, etc.), used as a breakdown dimension.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[Transaction type](https://xrpl.org/docs/references/protocol/transactions/types) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-type)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Transactions", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + } + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 310 + }, + "id": 118, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_economy{metric=\"transaction_rate\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Tx Rate\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Transaction Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Time to fetch a missing ledger from peers, at the 95th percentile.*\n\n###### How it's computed:\n*Per-acquire durations aggregated to their 95th percentile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; populated mainly during sync or back-fill.*\n\n###### Healthy range:\n*Low when synced; higher and more active while catching up.*\n\n###### Watch for:\n*A spike signals the node is falling behind or recovering from a fork.*\n\n###### Keywords:\n- **Ledger acquire (inbound fetch)** *(per node)* — fetching a specific missing ledger from peers; tracked by duration and outcome (complete/failed).\n- **Back-fill / catch-up** *(per node)* — fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Fork** *(network-wide)* — when nodes validate divergent ledger chains instead of a single agreed history.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Fork](https://xrpl.org/docs/concepts/consensus-protocol) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-acquire-inbound-fetch)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 0, + "y": 326 + }, + "id": 119, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire\"}[5m]))), \"series\", \"P95 Acquire\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger Acquire Duration (Inbound Fetch)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of completed ledger fetches split by outcome (complete or failed).*\n\n###### How it's computed:\n*Per-second rate of finished acquisitions grouped by outcome, per node, over a 5-minute window.*\n\n###### Reading it:\n*Complete should dominate; the failed line should stay near zero.*\n\n###### Healthy range:\n*Complete tracking fetch demand, failed near zero.*\n\n###### Watch for:\n*A rising failed rate means the node cannot fetch needed ledgers from its peers.*\n\n###### Keywords:\n- **Ledger acquire (inbound fetch)** *(per node)* — fetching a specific missing ledger from peers; tracked by duration and outcome (complete/failed).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-acquire-inbound-fetch)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Acquisitions / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 12, + "y": 326 + }, + "id": 120, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger Acquire Rate by Outcome", + "type": "timeseries" + }, + { + "title": "Job Queue Concurrency Limits", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 342 + }, + "collapsed": false, + "panels": [] + }, + { + "title": "Job Queue Saturation (Running vs Limit)", + "description": "###### What this is:\n*How close each concurrency-capped job type is to its ceiling. JobQueue enforces a per-type limit on how many jobs of that type may run at once, and the tight ones carry ledger-sync traffic: makeFetchPack 1, ledgerRequest 3, ledgerData 3, updatePaths 1, fetchTxnData 5. A type at its ceiling cannot start more work no matter how many workers are idle, so this is a different kind of limit from CPU or disk.*\n\n###### How it's computed:\n*Each jobq__running gauge divided by that type's own limit from JobTypes.h, so every line shares one 0-to-1 axis. 1.0 means running equals the limit. Multiply a reading by the limit shown in its legend to recover the raw job count. The gauges are published by JobQueue::collect() under the mutex that guards the counters.*\n\n###### Reading it:\n*Read the distance to 1.0, not the absolute height. Below 1.0 the type has spare slots and its queue wait is not the limit's fault. Touching 1.0 briefly is normal work. Sitting at 1.0 means the type is pinned at its ceiling and every further job of that type is being deferred rather than started, which is what turns into queue wait downstream. Because the limits differ, a raw count of 3 is saturation for ledgerRequest but only 60 percent for fetchTxnData; normalising is what makes the lines comparable.*\n\n###### Healthy range:\n*Below 1.0, with brief touches under load.*\n\n###### Watch for:\n*A line flat at 1.0: that type is the binding constraint. ledgerRequest pinned means the 3 slots shared by RcvGetLedger and RcvGetObjByHash are full, so peer ledger and object requests are queueing behind each other; the Ledger Data and Sync dashboard splits that wait by handler and shows the matching deferred depth. ledgerData or fetchTxnData pinned means inbound ledger data cannot be absorbed and validated ledger age will grow. makeFetchPack or updatePaths pinned at their limit of 1 means a single long job is blocking the whole type. Note the collector samples once per interval, so a line that never reaches 1.0 is not proof the type was never momentarily saturated.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::getNextJob (limit enforcement) / JobQueue::collect (publication)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 343 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(jobq_makefetchpack_running{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} / 1, \"series\", \"makeFetchPack (Limit 1)\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(jobq_ledgerrequest_running{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} / 3, \"series\", \"ledgerRequest (Limit 3)\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(jobq_ledgerdata_running{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} / 3, \"series\", \"ledgerData (Limit 3)\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(jobq_updatepaths_running{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} / 1, \"series\", \"updatePaths (Limit 1)\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(jobq_fetchtxndata_running{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} / 5, \"series\", \"fetchTxnData (Limit 5)\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "percentunit", + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "custom": { + "axisLabel": "Running / Limit", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3, + "thresholdsStyle": { + "mode": "line" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["node", "health"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "type", + "label": "Object Type", + "description": "Filter by internal object type (CountedObject class name)", + "type": "query", + "query": "label_values(object_count, type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "quantile", + "label": "Quantile", + "type": "custom", + "query": "0.5,0.75,0.95,0.99", + "current": { + "text": "0.95", + "value": "0.95" + }, + "options": [ + { + "text": "0.5", + "value": "0.5", + "selected": false + }, + { + "text": "0.75", + "value": "0.75", + "selected": false + }, + { + "text": "0.95", + "value": "0.95", + "selected": true + }, + { + "text": "0.99", + "value": "0.99", + "selected": false + } + ], + "includeAll": false, + "multi": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Node Health", + "uid": "node-health", + "refresh": "30s" +} diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json new file mode 100644 index 00000000000..34ccfd657d1 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -0,0 +1,1058 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Fine-grained breakdown of peer-to-peer overlay traffic beyond the main network view: squelch relay control, protocol overhead, validator-list distribution, transaction-set exchange, transaction availability, ledger-proof and replay traffic, and unclassified messages. \u2014 Use it to: Drill into individual overlay message categories to diagnose relay efficiency, overhead, and catch-up traffic.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Squelch Traffic (Messages)", + "description": "###### What this is:\n*Squelch relay-control messages in/out, plus messages suppressed by squelch and squelch directives that were ignored. Squelch reduces redundant message forwarding between peers.*\n\n###### How it's computed:\n*Per-second message rate for the squelch, squelch-suppressed, and squelch-ignored categories, in and out.*\n\n###### Reading it:\n*High suppressed counts mean squelch is saving bandwidth; ignored should stay low.*\n\n###### Healthy range:\n*workload-dependent; suppressed far above ignored.*\n\n###### Watch for:\n*High ignored counts (peers not honoring squelch) or squelch traffic itself dominating.*\n\n###### Keywords:\n- **Transaction suppression** *(per node)* \u2014 dropping a transaction already seen from another peer, so it is not reprocessed.\n- **Squelch** *(per node)* \u2014 control messages that tell a peer to stop forwarding a given validator's messages, cutting redundancy.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-suppression)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(squelch_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Squelch In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(squelch_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Squelch Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(squelch_suppressed_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Suppressed In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(squelch_suppressed_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Suppressed Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(squelch_ignored_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ignored In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(squelch_ignored_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ignored Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Overhead Traffic Breakdown (Bytes)", + "description": "###### What this is:\n*Overlay protocol overhead bytes split into base overhead, intra-cluster overhead, and validator-manifest distribution overhead.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the overhead, overhead-cluster, and overhead-manifest categories.*\n\n###### Reading it:\n*Base overhead is routine; cluster and manifest rise around cluster syncs and manifest changes.*\n\n###### Healthy range:\n*workload-dependent; low and stable.*\n\n###### Watch for:\n*Sustained high cluster or manifest overhead (frequent cluster state churn or manifest reissue).*\n\n###### Keywords:\n- **Overlay** *(per node)* \u2014 the peer-to-peer network layer over which nodes exchange transactions, proposals, and validations.\n- **Manifest** *(network-wide)* \u2014 a signed record binding a validator's rotating signing key to its stable master key.\n- **Cluster** *(cluster-wide)* \u2014 a group of trusted co-operated nodes that share load information and skip some verification.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Overlay](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) \u00b7 [Cluster](https://xrpl.org/docs/concepts/networks-and-servers/clustering) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(overhead_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Base Overhead In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(overhead_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Base Overhead Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(overhead_cluster_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Cluster In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(overhead_cluster_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Cluster Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(overhead_manifest_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Manifest In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(overhead_manifest_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Manifest Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Validator List Traffic", + "description": "###### What this is:\n*Bytes and messages exchanged distributing validator lists (trusted-list configuration) between peers.*\n\n###### How it's computed:\n*Per-second in/out byte and message rate for the validator-lists category.*\n\n###### Reading it:\n*Bursts when lists update or new peers connect; quiet otherwise.*\n\n###### Healthy range:\n*workload-dependent; occasional bursts.*\n\n###### Watch for:\n*Continuous high volume (repeated list re-fetching or churn).*\n\n###### Keywords:\n- **Validator list** *(network-wide)* \u2014 signed lists of recommended validators (UNLs) that peers distribute to each other.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Validator list](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validator-list)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validator_lists_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validator_lists_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validator_lists_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Messages In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validator_lists_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Messages Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Count", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Bytes/" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "binBps" + } + ] + } + ] + } + }, + { + "title": "Set Get/Share Traffic (Bytes)", + "description": "###### What this is:\n*Transaction-set fetch (get) and share bytes exchanged during ledger close.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the set-get and set-share categories.*\n\n###### Reading it:\n*Some exchange each ledger is normal as peers reconcile transaction sets.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High set-get (peers frequently missing transaction sets: possible sync delays).*\n\n###### Keywords:\n- **Set get/share** *(per node)* \u2014 exchange of candidate transaction sets between peers as they reconcile during a ledger close.\n- **Ledger close** *(network event)* \u2014 the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#set-get-share)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(set_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Set Get In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(set_get_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Set Get Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(set_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Set Share In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(set_share_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Set Share Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Have/Requested Transactions (Messages)", + "description": "###### What this is:\n*Transaction-availability messages: advertisements that a peer has certain transactions, and explicit requests for transaction data.*\n\n###### How it's computed:\n*Per-second in/out message rate for the have-transactions and requested-transactions categories.*\n\n###### Reading it:\n*Compare requested versus have to gauge how well transactions are propagating.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*Requested far exceeding have (peers behind on transaction propagation).*\n\n###### Keywords:\n- **Have / requested transactions** *(per node)* \u2014 advertisements that a peer holds certain transactions, and explicit requests for transaction data.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#have-requested-transactions)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(have_transactions_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Have TX In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(have_transactions_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Have TX Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(requested_transactions_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Requested TX In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(requested_transactions_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Requested TX Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Messages", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Unknown / Unclassified Traffic", + "description": "###### What this is:\n*Overlay traffic that matches no known message category, in bytes and messages.*\n\n###### How it's computed:\n*Current in/out byte and message counts for the unknown category.*\n\n###### Reading it:\n*Should be at or near zero.*\n\n###### Healthy range:\n*zero.*\n\n###### Watch for:\n*Any sustained non-zero value (protocol version mismatch, corrupted messages, or an unclassified new message type).*\n\n###### Keywords:\n- **Overlay** *(per node)* \u2014 the peer-to-peer network layer over which nodes exchange transactions, proposals, and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Overlay](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unknown_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Unknown Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unknown_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Unknown Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unknown_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Unknown Messages In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unknown_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Unknown Messages Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Count", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Bytes/" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] + } + }, + { + "title": "Proof Path Traffic", + "description": "###### What this is:\n*Proof-path request/response bytes used to verify individual ledger entries without downloading the whole ledger.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the proof-path request and response categories.*\n\n###### Reading it:\n*Rises when peers verify specific state, often during catch-up.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High sustained request volume (heavy state-verification load).*\n\n###### Keywords:\n- **Path request / discovery** *(per node)* \u2014 a client's ongoing pathfinding subscription (request) and the periodic path-refresh passes (discovery).\n- **Proof path** *(per node)* \u2014 messages that prove a single ledger entry exists without transferring the whole ledger.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Path request / discovery](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods/path-and-order-book-methods/path_find) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#path-request-discovery)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proof_path_request_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Request Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proof_path_request_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Request Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proof_path_response_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Response Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(proof_path_response_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Response Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Replay Delta Traffic", + "description": "###### What this is:\n*Replay-delta request/response bytes used to efficiently replay ledger state changes during catch-up.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the replay-delta request and response categories.*\n\n###### Reading it:\n*Active during catch-up and replay; quiet when synced.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*Continuous replay traffic (node repeatedly replaying rather than staying current).*\n\n###### Keywords:\n- **Replay delta** *(per node)* \u2014 messages carrying just the changes between ledgers, to replay state efficiently during catch-up.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#replay-delta)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(replay_delta_request_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Request Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(replay_delta_request_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Request Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(replay_delta_response_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Response Bytes In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(replay_delta_response_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Response Bytes Out\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "Bps", + "custom": { + "axisLabel": "Bytes", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject Handler (TMGetObjectByHash)", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "collapsed": false, + "panels": [] + }, + { + "title": "GetObject Handler Latency Breakdown", + "description": "###### What this is:\n*The three additive parts of TMGetObjectByHash service time, drawn on one axis so the expensive part names itself. Queue Wait is how long the RcvGetObjByHash job sat queued before a worker took it; its job type, ledgerRequest, allows only 3 to run at once. Handler Total is the whole job body once running. NodeStore Lookup is only the fetch loop inside that body. End-to-end service time is Queue Wait plus Handler Total, and Handler Total itself splits into NodeStore Lookup plus everything else.*\n\n###### How it's computed:\n*p99 of job_queued_us and job_running_us, both filtered to handler=\"RcvGetObjByHash\", plus p99 of getobject_lookup_us. Each is histogram_quantile over the microsecond bucket series, summed by le so the quantile is computed across the whole bucket set. The handler label is the sanitised addJob name, so RcvGetObjByHash is separated from RcvGetLedger even though both are job type ledgerRequest.*\n\n###### Reading it:\n*Read it as a subtraction, not as three independent lines. The vertical gap between Handler Total and NodeStore Lookup is the serialization and protobuf reply-building cost, because those are the only other things the job body does. So: Queue Wait tall with Handler Total flat means queue contention and the work itself is fine. Handler Total tracking NodeStore Lookup closely means storage is the bottleneck. Handler Total well above NodeStore Lookup means the cost has moved out of the fetch loop into reply construction. All three flat means this path is not the source of the slowness.*\n\n###### Healthy range:\n*All three sub-millisecond while peers ask for the usual 8 hashes per request; workload-dependent above that.*\n\n###### Watch for:\n*Queue Wait climbing while the other two stay flat: the ledgerRequest queue is saturated, so cross-check Job Queue Backlog and Deferred by Type and LedgerReq Wait by Handler on the Ledger Data and Sync dashboard to see which producer is starving it. A widening Handler Total minus NodeStore Lookup gap: reply construction regressed. NodeStore Lookup rising on its own: check getobject_lookups_total misses and the NuDB panels.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::recordGetObjectMetrics (lookup) / MetricsRegistry::recordJobStarted, recordJobFinished (queue, total)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 33 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{handler=\"RcvGetObjByHash\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"Queue Wait p99\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{handler=\"RcvGetObjByHash\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"Handler Total p99\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(getobject_lookup_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"NodeStore Lookup p99\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "µs", + "custom": { + "axisLabel": "Duration (μs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3, + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "GetObject Request Size Distribution", + "description": "###### What this is:\n*How many objects peers ask for per TMGetObjectByHash message, as a full distribution rather than an average. This characterises the request that caused any latency seen in the breakdown panel: large batches make the work genuinely large, which is a different problem from the same work becoming slower.*\n\n###### How it's computed:\n*Counts of requests falling in each object-count band per 5-minute window, from the getobject_request_objects bucket series, drawn as colour density.*\n\n###### Reading it:\n*A tight band at the bottom is honest traffic: InboundLedger asks for at most 8 hashes per call. Bands above 64 and above 1024 are the medium and large pricing bands, so mass there means the size surcharge is being applied. A hot cell in the top row is the overflow bucket and means requests larger than the top bucket boundary.*\n\n###### Healthy range:\n*Nearly all mass in the lowest bands (8 objects or fewer per request).*\n\n###### Watch for:\n*Mass appearing in the high bands, especially a persistent hot row near the top: a peer is batching thousands of hashes per message, which is what the differential pricing exists to charge for. Confirm with GetObject Charge Distribution and GetObject Rejections. Buckets are explicit (1,2,4,8,16,64,256,1024,4096,12288) and reach the handler's hard cap, so the top row is real traffic at the cap, not a measurement ceiling.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::recordGetObjectMetrics`", + "type": "heatmap", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "yAxis": { + "axisLabel": "Objects Per Request", + "unit": "short" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum(increase(getobject_request_objects_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m])) by (le)", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "title": "GetObject Lookups by Result", + "description": "###### What this is:\n*NodeStore lookups performed by the handler, split into hits and misses. A miss does a node-store seek while a hit is usually served from cache, so the hit/miss mix is the reason NodeStore Lookup time moves.*\n\n###### How it's computed:\n*Per-second rate of getobject_lookups_total, grouped by the result label. The counter is advanced once per request with the batch total, not once per object, so the rate is objects per second rather than requests per second.*\n\n###### Reading it:\n*Use this to explain the NodeStore Lookup line on the breakdown panel. A miss-heavy mix makes that line rise for a real reason: seeks, not a regression. A hit-heavy mix with rising lookup time points at the storage layer instead.*\n\n###### Healthy range:\n*Hits dominating on a warm synced node; misses low and driven by genuine catch-up requests.*\n\n###### Watch for:\n*A sustained miss rate far above the hit rate: a peer is asking for hashes this node does not hold, which is either a badly out-of-sync peer or probing traffic. Cross-check GetObject Charge Distribution, since misses are billed first and at eight times the hit cost.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::recordGetObjectMetrics`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(sum by (result, service_instance_id, xrpl_branch, xrpl_node_role) (rate(getobject_lookups_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", result=~\"$result\"}[$__rate_interval])), \"series\", \"$1\", \"result\", \"(.*)\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "cps", + "custom": { + "axisLabel": "Lookups / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject Rejections", + "description": "###### What this is:\n*Requests refused by the message handler before any NodeStore access, split by which gate refused them: oversize means more objects than the handler accepts, malformed_ledgerhash means the ledger hash was not 32 bytes.*\n\n###### How it's computed:\n*Per-second rate of getobject_rejected_total, grouped by the reason label. Both gates run in onMessage before the job is queued, so a rejection consumes no queue slot and no NodeStore lookup.*\n\n###### Reading it:\n*Any non-zero value is traffic that does not conform to the protocol: an honest peer asks for at most 8 hashes and always sends a full-size hash. Because the gates fire before the fetch loop, rejections explain why request volume can be high while lookups stay flat.*\n\n###### Healthy range:\n*Zero. No honest peer produces either rejection.*\n\n###### Watch for:\n*A rising oversize rate: a peer is probing the request-size limit. Confirm the pricing response on GetObject Charge Distribution, and expect the peer to be disconnected once its resource balance crosses the drop threshold. A rising malformed rate points at a broken or hostile client rather than at load.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage (TMGetObjectByHash)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 49 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_node_role) (rate(getobject_rejected_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$reason\"}[$__rate_interval])), \"series\", \"$1\", \"reason\", \"(.*)\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "cps", + "custom": { + "axisLabel": "Rejections / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject Charge Distribution", + "description": "###### What this is:\n*The dynamic resource charge applied per TMGetObjectByHash request, as percentiles. This is the differential-pricing component only, so it shows whether cost actually escalates with request size and miss ratio the way the pricing model intends.*\n\n###### How it's computed:\n*p50, p90 and p99 of getobject_charge over the dashboard rate interval, from its bucket series summed by le. The value recorded is the charge that was applied, computed from billable hits, billable misses and the request-size band.*\n\n###### Reading it:\n*p50 sitting at zero is the healthy shape: requests inside the free allowance cost nothing. Movement in p99 while p50 stays at zero means a small number of expensive requests, which is exactly the traffic the model is meant to price. Compare with GetObject Request Size Distribution: charge should rise in steps as requests cross the size-band edges, not smoothly.*\n\n###### Healthy range:\n*p50 at zero, p99 low. Honest requests of 16 objects or fewer are free by design.*\n\n###### Watch for:\n*p99 climbing steadily: sustained expensive traffic, and the peers producing it should be approaching the resource drop threshold. Buckets are explicit and bracket the resource thresholds (5000 warning, 25000 drop), so p99 crossing 25000 means senders are being disconnected on a single message. The axis is deliberately unscaled rather than abbreviated, so those two numbers are readable exactly rather than as 5 K and 25 K.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::computeGetObjectByHashFee`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 49 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(histogram_quantile(0.5, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(getobject_charge_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"Charge p50\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(histogram_quantile(0.9, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(getobject_charge_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"Charge p90\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(getobject_charge_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"Charge p99\", \"\", \"\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "none", + "custom": { + "axisLabel": "Charge (Cost Units)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3, + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["network", "peer"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "result", + "label": "Lookup Result", + "description": "Filter GetObject NodeStore lookups by result [hit / miss]", + "type": "query", + "query": "label_values(getobject_lookups_total, result)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "reason", + "label": "Rejection Reason", + "description": "Filter GetObject rejections by gate [oversize / malformed_ledgerhash]", + "type": "query", + "query": "label_values(getobject_rejected_total, reason)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Overlay Traffic Detail", + "uid": "overlay-traffic-detail", + "refresh": "30s" +} diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json new file mode 100644 index 00000000000..4e0c5528fe2 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -0,0 +1,495 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Consensus proposals and validations received from peers, and transaction relay efficiency for this node. \u2014 Use it to: Check that peers are feeding this node proposals, validations, and transactions, and gauge relay overhead.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Peer Proposal Receive Rate", + "description": "###### What this is:\n*How many consensus proposals this node receives from peers per second.*\n\n###### How it's computed:\n*Per-second rate of received proposals over 5 minutes, per node.*\n\n###### Reading it:\n*A steady rate roughly proportional to the number of proposing validators.*\n\n###### Healthy range:\n*Workload-dependent; scales with validator count and connectivity.*\n\n###### Watch for:\n*A drop toward zero (isolation from the network) or a sudden flood far above baseline (proposal spam).*\n\n###### Keywords:\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Proposers** *(network event)* \u2014 the count of validators whose proposals this node heard in the last closed round.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMProposeSet)`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"peer.proposal.receive\"}[$__rate_interval])), \"series\", \"Proposals Received / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Proposals / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Peer Validation Receive Rate", + "description": "###### What this is:\n*How many ledger validations this node receives from peers per second.*\n\n###### How it's computed:\n*Per-second rate of received validations over 5 minutes, per node.*\n\n###### Reading it:\n*Steady and proportional to the number of validators the node hears from.*\n\n###### Healthy range:\n*Workload-dependent; roughly one burst per validator per closed ledger.*\n\n###### Watch for:\n*A fall toward zero (loss of validator connectivity) or an abnormal surge from untrusted sources.*\n\n###### Keywords:\n- **Trusted / untrusted / duplicate** *(per node)* \u2014 message classification: trusted (from UNL validators), untrusted (others), or duplicate (already seen).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMValidation)`\n\n###### References:\n[Trusted / untrusted / duplicate](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#trusted-untrusted-duplicate)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"peer.validation.receive\"}[$__rate_interval])), \"series\", \"Validations Received / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Validations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Proposals Trusted vs Untrusted", + "description": "###### What this is:\n*The share of received proposals that come from trusted (UNL) validators versus everyone else.*\n\n###### How it's computed:\n*Received-proposal rate split by trust status over 5 minutes, shown as proportions.*\n\n###### Reading it:\n*A healthy node with a good UNL shows a solid trusted slice.*\n\n###### Healthy range:\n*Workload-dependent; a meaningful trusted fraction is expected.*\n\n###### Watch for:\n*A collapsing trusted share or a large untrusted volume, consistent with proposal flooding from non-UNL peers.*\n\n###### Keywords:\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Trusted / untrusted / duplicate** *(per node)* \u2014 message classification: trusted (from UNL validators), untrusted (others), or duplicate (already seen).\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMProposeSet)`\n\n###### References:\n[Proposal](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Trusted / untrusted / duplicate](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#proposal)", + "type": "piechart", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (proposal_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"proposal_trusted\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + } + }, + { + "title": "Validations Trusted vs Untrusted", + "description": "###### What this is:\n*The share of received validations from trusted (UNL) validators versus untrusted sources.*\n\n###### How it's computed:\n*Received-validation rate split by trust status over 5 minutes, shown as proportions.*\n\n###### Reading it:\n*The trusted slice should dominate for a well-configured node.*\n\n###### Healthy range:\n*Workload-dependent; trusted validations expected to be the majority.*\n\n###### Watch for:\n*A shrinking trusted share or a spike of untrusted validations, which can indicate misconfiguration or abuse.*\n\n###### Keywords:\n- **Trusted / untrusted / duplicate** *(per node)* \u2014 message classification: trusted (from UNL validators), untrusted (others), or duplicate (already seen).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMValidation)`\n\n###### References:\n[Trusted / untrusted / duplicate](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#trusted-untrusted-duplicate)", + "type": "piechart", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (validation_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"validation_trusted\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + } + }, + { + "title": "Reduce-Relay Peer Selection", + "description": "###### What this is:\n*How transaction relay picks peers: chosen relay sources, suppressed peers, and peers with the feature off.*\n\n###### How it's computed:\n*Current peer counts in each category (selected, suppressed, not-enabled), per node.*\n\n###### Reading it:\n*A high suppressed-to-selected ratio means relay is saving bandwidth as intended.*\n\n###### Healthy range:\n*Workload-dependent; suppressed should exceed selected in a well-connected mesh.*\n\n###### Watch for:\n*A large not-enabled count (older peers forcing full relay) or selected climbing while suppressed falls.*\n\n###### Keywords:\n- **Reduce-relay** *(per node)* \u2014 an optimization that relays messages through selected peers only, suppressing redundant forwarding.\n- **Transaction suppression** *(per node)* \u2014 dropping a transaction already seen from another peer, so it is not reprocessed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerReduceRelayGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#reduce-relay)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(reduce_relay_metrics{metric=\"selected_peers\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Selected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(reduce_relay_metrics{metric=\"suppressed_peers\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Suppressed\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(reduce_relay_metrics{metric=\"not_enabled_peers\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Not Enabled\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Peer Count", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Reduce-Relay Missing-Tx Frequency", + "description": "###### What this is:\n*How often a peer has to fetch a transaction it missed because relay suppressed it.*\n\n###### How it's computed:\n*The reported frequency of on-demand missing-transaction fetches, per node.*\n\n###### Reading it:\n*Lower is better; near-flat means suppression is well tuned.*\n\n###### Healthy range:\n*Workload-dependent; a low, stable value is expected.*\n\n###### Watch for:\n*A rising trend, meaning suppression is too aggressive and the on-demand fetch path is growing.*\n\n###### Keywords:\n- **Reduce-relay** *(per node)* \u2014 an optimization that relays messages through selected peers only, suppressing redundant forwarding.\n- **Transaction suppression** *(per node)* \u2014 dropping a transaction already seen from another peer, so it is not reprocessed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerReduceRelayGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#reduce-relay)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(reduce_relay_metrics{metric=\"missing_tx_freq\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Missing Tx Freq\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Frequency", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["network", "peer"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "proposal_trusted", + "label": "Proposal Trusted", + "description": "Filter by proposal trust status (true = from trusted validator)", + "type": "query", + "query": "label_values(span_calls_total{span_name=\"peer.proposal.receive\"}, proposal_trusted)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "validation_trusted", + "label": "Validation Trusted", + "description": "Filter by validation trust status (true = from trusted validator)", + "type": "query", + "query": "label_values(span_calls_total{span_name=\"peer.validation.receive\"}, validation_trusted)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Peer Network", + "uid": "peer-network", + "refresh": "30s" +} diff --git a/docker/telemetry/grafana/dashboards/peer-quality.json b/docker/telemetry/grafana/dashboards/peer-quality.json new file mode 100644 index 00000000000..aa51f07afef --- /dev/null +++ b/docker/telemetry/grafana/dashboards/peer-quality.json @@ -0,0 +1,573 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Peer connection quality: latency, divergence, version distribution and upgrade guidance, resource disconnects, and inbound/outbound balance. \u2014 Use it to: Assess how good this node's peer set is and whether connectivity or an upgrade needs attention.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "P90 Peer Latency", + "description": "###### What this is:\n*90th-percentile round-trip latency to connected peers, in milliseconds.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the p90 peer latency.*\n\n###### Reading it:\n*Lower is better; green under 200ms, yellow to 500ms, red above.*\n\n###### Healthy range:\n*Under 200ms.*\n\n###### Watch for:\n*Rising latency, which points to network congestion or geographically distant, poorly performing peers.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_quality{metric=\"peer_latency_p90_ms\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"P90 Latency\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 200 + }, + { + "color": "red", + "value": 500 + } + ] + } + }, + "overrides": [] + } + }, + { + "title": "Insane/Diverged Peers", + "description": "###### What this is:\n*Count of connected peers whose ledger state has diverged from the network.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the diverged-peer count.*\n\n###### Reading it:\n*Zero is healthy; any count means those peers disagree on ledger state.*\n\n###### Healthy range:\n*0 diverged peers.*\n\n###### Watch for:\n*A persistent non-zero count, which can indicate peers on a fork or misbehaving peers.*\n\n###### Keywords:\n- **Insane / diverged peers** *(per node)* \u2014 connected peers whose ledger state disagrees with the network \u2014 possibly on a fork or misbehaving.\n- **Fork** *(network-wide)* \u2014 when nodes validate divergent ledger chains instead of a single agreed history.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Fork](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#insane-diverged-peers)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_quality{metric=\"peers_insane_count\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Insane Peers\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Higher Version Peers %", + "description": "###### What this is:\n*Percentage of connected peers running a newer rippled version than this node.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the higher-version peer percentage.*\n\n###### Reading it:\n*A high percentage suggests this node is behind and should be upgraded.*\n\n###### Healthy range:\n*Under 30%.*\n\n###### Watch for:\n*A majority of peers on a newer version, a strong upgrade signal.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_quality{metric=\"peers_higher_version_pct\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Higher Version %\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "red", + "value": 60 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Upgrade Recommended", + "description": "###### What this is:\n*A flag indicating whether an upgrade is advised based on peer version analysis (Yes/No).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the upgrade-recommended flag.*\n\n###### Reading it:\n*No is healthy; Yes means most peers run a newer version.*\n\n###### Healthy range:\n*No.*\n\n###### Watch for:\n*A Yes state, indicating the node risks falling out of step with the network.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_quality{metric=\"upgrade_recommended\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Upgrade\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "No", + "color": "green" + } + } + }, + { + "type": "value", + "options": { + "1": { + "text": "Yes", + "color": "red" + } + } + } + ], + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Resource Disconnects", + "description": "###### What this is:\n*Cumulative count of peers dropped for exceeding resource (load) limits.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the resource-disconnect total over time.*\n\n###### Reading it:\n*A flat line is healthy; a rising line means peers are being dropped for overuse.*\n\n###### Healthy range:\n*Flat / near constant.*\n\n###### Watch for:\n*A steep climb, which flags aggressive or misbehaving peers being shed as backpressure.*\n\n###### Keywords:\n- **Resource disconnect** *(per node)* \u2014 a peer dropped for exceeding resource/load limits \u2014 the node shedding abusive or overactive peers.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(server_info{metric=\"peer_disconnects_resources\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Disconnects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Disconnects", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "Inbound vs Outbound Peers", + "description": "###### What this is:\n*Active inbound versus outbound peer connection counts.*\n\n###### How it's computed:\n*Instantaneous gauge readings of active inbound and outbound peer counts.*\n\n###### Reading it:\n*A balanced mix is healthy for connectivity and resilience.*\n\n###### Healthy range:\n*A mix of both; some outbound connections present.*\n\n###### Watch for:\n*All-inbound with no outbound, which usually means NAT or firewall issues block outbound peering.*\n\n###### Source:\n[PeerfinderManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/PeerfinderManager.cpp)\n\n###### Function:\n`Logic Stats ctor`", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_finder_active_inbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Inbound\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_finder_active_outbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Outbound\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "Inbound.*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "Outbound.*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + } + ] + } + } + ], + "schemaVersion": 39, + "tags": ["network", "peer"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Peer Quality", + "uid": "peer-quality", + "refresh": "30s" +} diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json new file mode 100644 index 00000000000..313e4ef47f9 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -0,0 +1,794 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: RPC request throughput and response timing, pathfinding cost, gRPC read traffic, and resource-manager enforcement for this node. \u2014 Use it to: Watch client request load and cost, and confirm resource limits are protecting the node.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "RPC Request Rate", + "description": "###### What this is:\n*How many RPC requests the server counts per second.*\n\n###### How it's computed:\n*Per-second rate of the RPC request counter over 5 minutes, per node.*\n\n###### Reading it:\n*A steady line proportional to client demand; cross-checks the trace-based RPC dashboard.*\n\n###### Healthy range:\n*Workload-dependent; tracks client activity.*\n\n###### Watch for:\n*A sudden surge above baseline, consistent with a client flooding the RPC endpoint.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(rpc_requests_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Requests / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "title": "RPC Response Time", + "description": "###### What this is:\n*The 95th-percentile end-to-end RPC response time, including HTTP handling.*\n\n###### How it's computed:\n*95th-percentile of measured response times over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; this is broader than command-only latency.*\n\n###### Healthy range:\n*A few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A sustained climb, indicating handler overload or slow downstream work.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Response Time\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "RPC Response Size", + "description": "\u26a0 Instrument mismatch \u2014 values unreliable. Response size is recorded through the millisecond-scaled event histogram (rpc_size_milliseconds_bucket), so byte values saturate at the top time bucket (5000) and the percentiles are not true byte sizes. A dedicated byte-unit histogram is needed to fix this; tracked separately. Treat this panel as indicative only until then.\n\n###### What this is:\n*The 95th-percentile size of RPC response payloads in bytes.*\n\n###### How it's computed:\n*95th-percentile of response payload sizes over the dashboard rate interval, per node.*\n\n###### Reading it:\n*Smaller is cheaper; large responses cost bandwidth and memory.*\n\n###### Healthy range:\n*Workload-dependent; small for status queries, large for bulk data queries.*\n\n###### Watch for:\n*Growth in large responses, consistent with expensive queries or API misuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_size_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Response Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "decbytes", + "custom": { + "axisLabel": "Size (Bytes)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "RPC Response Time Distribution", + "description": "###### What this is:\n*RPC response time at the 90th, 95th, and 99th percentiles together.*\n\n###### How it's computed:\n*Three response-time quantiles over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; the gap between P90 and P99 shows the long tail.*\n\n###### Healthy range:\n*Percentiles clustered low; workload-dependent.*\n\n###### Watch for:\n*A widening P99, revealing long-tail or bimodal slow requests.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.9, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P90\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P99\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Pathfinding Fast Duration", + "description": "###### What this is:\n*The 95th-percentile time of the fast pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of fast pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; fast mode trades accuracy for speed.*\n\n###### Healthy range:\n*Typically a few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A rising trend, indicating heavy pathfinding demand or an expensive order book.*\n\n###### Keywords:\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n- **Order book** *(network-wide)* \u2014 the ledger's list of standing offers to trade a currency pair on the decentralized exchange.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Order book](https://xrpl.org/docs/concepts/tokens/decentralized-exchange) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#pathfinding-fast-full)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Fast Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Pathfinding Full Duration", + "description": "###### What this is:\n*The 95th-percentile time of the full, exhaustive pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of full pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; full mode is much more expensive than fast mode.*\n\n###### Healthy range:\n*Tens to hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained high durations, consistent with pathfinding-heavy clients straining the node.*\n\n###### Keywords:\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#pathfinding-fast-full)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Full Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Resource Warnings Rate", + "description": "###### What this is:\n*How often the resource manager warns a peer or client for excessive usage, per second.*\n\n###### How it's computed:\n*Per-second rate of resource-warning events over 5 minutes, per node.*\n\n###### Reading it:\n*Green near zero, yellow above 0.1/sec, red above 1/sec.*\n\n###### Healthy range:\n*Near zero in normal operation.*\n\n###### Watch for:\n*A rising rate, consistent with aggressive clients that may need throttling.*\n\n###### Keywords:\n- **Resource drops / warnings** *(per node)* \u2014 the resource manager warning (then dropping/blocking) a peer or client for excessive usage.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/resource/detail/Logic.h)\n\n###### Function:\n`Logic::Stats ctor`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-drops-warnings)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(warn_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Warnings / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + } + }, + { + "title": "Resource Drops Rate", + "description": "###### What this is:\n*How often the resource manager drops or blocks a peer or client for abuse, per second.*\n\n###### How it's computed:\n*Per-second rate of resource-drop events over 5 minutes, per node.*\n\n###### Reading it:\n*Green near zero, yellow above 0.01/sec, red above 0.1/sec.*\n\n###### Healthy range:\n*Zero when no abusive consumers are present.*\n\n###### Watch for:\n*Non-zero values, meaning the node is actively rejecting abusive connections.*\n\n###### Keywords:\n- **Resource drops / warnings** *(per node)* \u2014 the resource manager warning (then dropping/blocking) a peer or client for excessive usage.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/resource/detail/Logic.h)\n\n###### Function:\n`Logic::Stats ctor`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-drops-warnings)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(drop_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Drops / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.1 + } + ] + } + }, + "overrides": [] + } + }, + { + "title": "gRPC Request Rate by Method (Spans)", + "description": "###### What this is:\n*How many gRPC calls of each method the node serves per second.*\n\n###### How it's computed:\n*Per-second rate of gRPC calls over 5 minutes, grouped by method and node.*\n\n###### Reading it:\n*Non-zero only when reporting/Clio-style clients use the gRPC API.*\n\n###### Healthy range:\n*Workload-dependent; zero without gRPC traffic.*\n\n###### Watch for:\n*A single method spiking, consistent with a heavy or misbehaving gRPC consumer.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n- **Clio / reporting client** *(per node)* \u2014 an external read-scaling service (Clio) that serves history/API queries, often via the gRPC interface.\n- **gRPC** *(per node)* \u2014 a binary RPC interface used mainly by reporting/Clio-style clients, separate from HTTP/WebSocket.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[GRPCServer.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/GRPCServer.cpp)\n\n###### Function:\n`GRPCServerImpl::CallData::process`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (method, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$grpc_method\", span_name=~\"grpc\\\\..*\"}[$__rate_interval])), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Calls / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "gRPC Latency P95 by Method (Spans)", + "description": "###### What this is:\n*The 95th-percentile latency of each gRPC method.*\n\n###### How it's computed:\n*95th-percentile of gRPC call durations over 5 minutes, grouped by method and node.*\n\n###### Reading it:\n*Lower is better; identifies slow gRPC read paths.*\n\n###### Healthy range:\n*Workload-dependent; scales with ledger data volume served.*\n\n###### Watch for:\n*A method whose latency climbs, indicating an expensive read path.*\n\n###### Keywords:\n- **gRPC** *(per node)* \u2014 a binary RPC interface used mainly by reporting/Clio-style clients, separate from HTTP/WebSocket.\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[GRPCServer.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/GRPCServer.cpp)\n\n###### Function:\n`GRPCServerImpl::CallData::process`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#grpc)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, method, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$grpc_method\", span_name=~\"grpc\\\\..*\"}[5m]))), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "gRPC Error Rate by Status (Spans)", + "description": "###### What this is:\n*gRPC call rate broken down by outcome status (success or error).*\n\n###### How it's computed:\n*Per-second rate of gRPC calls over 5 minutes, grouped by status and node.*\n\n###### Reading it:\n*Success should dominate; the error rate should stay low.*\n\n###### Healthy range:\n*Workload-dependent; errors near the floor.*\n\n###### Watch for:\n*A rising error rate, indicating gRPC clients hitting failures.*\n\n###### Keywords:\n- **gRPC** *(per node)* \u2014 a binary RPC interface used mainly by reporting/Clio-style clients, separate from HTTP/WebSocket.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[GRPCServer.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/GRPCServer.cpp)\n\n###### Function:\n`GRPCServerImpl::CallData::process`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#grpc)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (grpc_status, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=~\"grpc\\\\..*\", grpc_status!=\"\"}[$__rate_interval])), \"series\", \"$1\", \"grpc_status\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Calls / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Pathfinding Compute Duration (Spans)", + "description": "###### What this is:\n*The 95th-percentile time to compute paths for a single request.*\n\n###### How it's computed:\n*95th-percentile of path-computation durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; complements the fast/full pathfinding timers with span-level detail.*\n\n###### Healthy range:\n*Workload-dependent; grows with order-book depth and request complexity.*\n\n###### Watch for:\n*A rising trend under pathfinding load, indicating expensive path computation.*\n\n###### Keywords:\n- **Batch vs single RPC** *(per node)* \u2014 whether an RPC request carried several commands (batch) or one (single); batch is used by bulk clients.\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n- **Order book** *(network-wide)* \u2014 the ledger's list of standing offers to trade a currency pair on the decentralized exchange.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PathRequest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequest.cpp)\n\n###### Function:\n`PathRequest::doUpdate`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Order book](https://xrpl.org/docs/concepts/tokens/decentralized-exchange) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#batch-vs-single-rpc)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"pathfind.compute\"}[5m]))), \"series\", \"P95 Compute\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "Pathfinding Request & Discovery Rate (Spans)", + "description": "###### What this is:\n*The rate of client path requests versus path-discovery passes.*\n\n###### How it's computed:\n*Per-second rates of path requests and discovery passes over 5 minutes, per node.*\n\n###### Reading it:\n*Discovery cost tracks request demand; useful for subscription-heavy nodes.*\n\n###### Healthy range:\n*Workload-dependent; both zero without pathfinding clients.*\n\n###### Watch for:\n*Discovery rate climbing out of proportion to requests, a cost driver for subscription-heavy nodes.*\n\n###### Keywords:\n- **Path request / discovery** *(per node)* \u2014 a client's ongoing pathfinding subscription (request) and the periodic path-refresh passes (discovery).\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PathFind.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/handlers/orderbook/PathFind.cpp) \u00b7 [PathRequest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequest.cpp)\n\n###### Function:\n`doPathFind ; PathRequest::findPaths`\n\n###### References:\n[Path request / discovery](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods/path-and-order-book-methods/path_find) \u00b7 [Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#path-request-discovery)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"pathfind.request\"}[$__rate_interval])), \"series\", \"Requests / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"pathfind.discover\"}[$__rate_interval])), \"series\", \"Discoveries / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["rpc", "pathfinding"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "grpc_method", + "label": "gRPC Method", + "description": "Filter by gRPC method (GetLedger, GetLedgerData, GetLedgerDiff, GetLedgerEntry)", + "type": "query", + "query": "label_values(span_calls_total{span_name=~\"grpc\\\\..*\"}, method)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "RPC & Pathfinding", + "uid": "rpc-pathfinding", + "refresh": "30s" +} diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json new file mode 100644 index 00000000000..86b22c806e9 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -0,0 +1,1115 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "id": 11, + "type": "row", + "title": "Aggregate RPC (all commands)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "title": "RPC Request Rate by Command", + "description": "###### What this is:\n*How many times each RPC command runs per second.*\n\n###### How it's computed:\n*Per-second rate of command executions over 5 minutes, grouped by command name and node.*\n\n###### Reading it:\n*Compare bars/lines across commands to see which endpoints drive load.*\n\n###### Healthy range:\n*Workload-dependent; dominated by whichever commands clients call most.*\n\n###### Watch for:\n*A single command spiking far above its norm, consistent with a client hammering one endpoint.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])), \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "reqps", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 1 + }, + { + "title": "RPC Latency P95 by Command", + "description": "###### What this is:\n*The 95th-percentile response time for each RPC command.*\n\n###### How it's computed:\n*95th-percentile of per-command execution durations over 5 minutes, grouped by command and node.*\n\n###### Reading it:\n*Lower is better; watch the slowest commands.*\n\n###### Healthy range:\n*Fast lookups a few ms; heavy queries tens to hundreds of ms. Workload-dependent.*\n\n###### Watch for:\n*A command whose latency climbs steadily, pointing to an expensive or degrading query path.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m]))), \"series\", \"P95 $1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 2 + }, + { + "title": "RPC Error Rate", + "description": "###### What this is:\n*The percentage of each command's calls that finished with an error.*\n\n###### How it's computed:\n*Error calls divided by total calls per command over 5 minutes, expressed as a percent.*\n\n###### Reading it:\n*Green under 1%, yellow 1-5%, red above 5%.*\n\n###### Healthy range:\n*Below 1% per command in normal operation.*\n\n###### Watch for:\n*A command sitting red, indicating malformed input, overload, or a broken handler.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) / sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])) * 100, \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percent", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "id": 3 + }, + { + "title": "RPC Latency Heatmap", + "description": "###### What this is:\n*The full distribution of RPC response times over the window.*\n\n###### How it's computed:\n*Counts of requests in each latency band per 5-minute window, shown as color density.*\n\n###### Reading it:\n*A tight low band is healthy; multiple bands mean mixed fast/slow requests.*\n\n###### Healthy range:\n*Most mass in the low-latency bands; workload-dependent.*\n\n###### Watch for:\n*A distinct high-latency cluster (bimodal behavior) that percentiles alone would hide.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "heatmap", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "yAxis": { + "axisLabel": "Duration (ms)", + "unit": "ms" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])) by (le)", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + } + }, + "id": 4 + }, + { + "title": "Overall RPC Throughput", + "description": "###### What this is:\n*Two layers of the request pipeline: connections accepted by the HTTP handler versus requests parsed and dispatched.*\n\n###### How it's computed:\n*Per-second rates of the outer request-accept layer and the inner processing layer over 5 minutes, per node.*\n\n###### Reading it:\n*The two lines should track each other closely.*\n\n###### Healthy range:\n*Workload-dependent; both lines roughly equal.*\n\n###### Watch for:\n*A gap between accept and process, meaning requests are queued or rejected before dispatch.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler::processSession ; ServerHandler::processRequest`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=\"rpc.http_request\"}[$__rate_interval])), \"series\", \"rpc.http_request / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=\"rpc.process\"}[$__rate_interval])), \"series\", \"rpc.process / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "reqps", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 5 + }, + { + "title": "RPC Success vs Error", + "description": "###### What this is:\n*Aggregate rate of successful versus failed RPC commands across all command types.*\n\n###### How it's computed:\n*Per-second rates of ok-status and error-status command completions over 5 minutes, per node.*\n\n###### Reading it:\n*Successful commands report an ok status and should dominate; the error line should stay near the floor.*\n\n###### Healthy range:\n*Error line near zero relative to success; workload-dependent.*\n\n###### Watch for:\n*A sustained error line, which warrants drilling into the per-command breakdown.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\", status_code=\"STATUS_CODE_OK\"}[$__rate_interval])), \"series\", \"Success\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])), \"series\", \"Error\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Commands / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 6 + }, + { + "title": "Top Commands by Volume", + "description": "###### What this is:\n*The ten most-called RPC commands over the recent window.*\n\n###### How it's computed:\n*Total invocation counts per command over the last 5 minutes, ranked and capped at ten.*\n\n###### Reading it:\n*The top bars are the hottest endpoints driving node load.*\n\n###### Healthy range:\n*Workload-dependent; typically led by lightweight status queries.*\n\n###### Watch for:\n*An unexpected command dominating the ranking, consistent with automated abuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))), \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short" + }, + "overrides": [] + }, + "id": 7 + }, + { + "title": "WebSocket Message Rate", + "description": "###### What this is:\n*How many RPC messages arrive over WebSocket connections per second.*\n\n###### How it's computed:\n*Per-second rate of processed WebSocket RPC messages over 5 minutes, per node.*\n\n###### Reading it:\n*Non-zero only when clients use WebSocket; zero is normal for HTTP-only nodes.*\n\n###### Healthy range:\n*Workload-dependent; zero if no WebSocket clients.*\n\n###### Watch for:\n*A sudden surge from a single source, consistent with a chatty or abusive subscription client.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n- **WebSocket** *(per node)* \u2014 a persistent-connection API transport used by subscription clients; zero on HTTP-only nodes.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler::processSession`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [WebSocket](https://xrpl.org/docs/references/http-websocket-apis/api-conventions) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval])), \"series\", \"WS Messages / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops" + }, + "overrides": [] + }, + "id": 8 + }, + { + "title": "RPC Resource Cost by Command", + "description": "###### What this is:\n*RPC traffic grouped by resource-cost category rather than by command name.*\n\n###### How it's computed:\n*Per-second rate of commands over 5 minutes, grouped by load/cost category.*\n\n###### Reading it:\n*Cheap categories should dominate; costly ones should be a small slice.*\n\n###### Healthy range:\n*Workload-dependent; low-cost categories carry most traffic.*\n\n###### Watch for:\n*Growth in high-cost categories such as exception or malformed requests, which point to problematic clients.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, load_type, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=~\"rpc.command.*\", load_type!=\"\"}[$__rate_interval])), \"series\", \"$1\", \"load_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 9 + }, + { + "title": "Batch vs Single RPC Requests", + "description": "###### What this is:\n*The rate of batched RPC requests versus single-command requests.*\n\n###### How it's computed:\n*Per-second rates of batch and single requests over 5 minutes, per node.*\n\n###### Reading it:\n*Single requests usually dominate; batch is used by bulk clients.*\n\n###### Healthy range:\n*Workload-dependent; batch typically a minority.*\n\n###### Watch for:\n*A batch rate climbing sharply, consistent with bulk automation or amplification attempts.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n- **Batch vs single RPC** *(per node)* \u2014 whether an RPC request carried several commands (batch) or one (single); batch is used by bulk clients.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler::processRequest`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"rpc.process\", is_batch=\"true\"}[$__rate_interval])), \"series\", \"Batch\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"rpc.process\", is_batch=\"false\"}[$__rate_interval])), \"series\", \"Single\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 10 + }, + { + "id": 12, + "type": "row", + "title": "Per-Method Detail", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "panels": [] + }, + { + "title": "RPC Call Rate (All Methods)", + "description": "###### What this is:\n*Overall rate of RPC method calls that started, finished, and errored, across all methods.*\n\n###### How it's computed:\n*Per-second rate of each counter over a 5-minute window, summed per node.*\n\n###### Reading it:\n*Started should closely track finished; errored should be a small fraction.*\n\n###### Healthy range:\n*Workload-dependent; started \u2248 finished, errored near zero.*\n\n###### Watch for:\n*A growing gap between started and finished (calls hanging), or an errored line that rises with load.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcStarted / recordRpcFinished / recordRpcErrored`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 51 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"Started/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"Finished/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_errored_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"Errored/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 13 + }, + { + "title": "Per-Method Call Rate (Top 10)", + "description": "###### What this is:\n*The ten busiest RPC methods by call rate.*\n\n###### How it's computed:\n*Per-second start rate over 5 minutes, per method, showing the top ten.*\n\n###### Reading it:\n*Identifies which methods dominate load; the mix shifts with client behaviour.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A single method suddenly dominating, which can signal a runaway client or abusive query pattern.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcStarted`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 59 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, rate(rpc_method_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 14 + }, + { + "title": "Per-Method Error Rate (Top 10)", + "description": "###### What this is:\n*The ten RPC methods producing the most errors.*\n\n###### How it's computed:\n*Per-second error rate over 5 minutes, per method, top ten.*\n\n###### Reading it:\n*Normally near zero; persistent errors point to a specific failing method.*\n\n###### Healthy range:\n*Near zero for well-behaved traffic.*\n\n###### Watch for:\n*Sustained errors concentrated on one method \u2014 a broken client, a bad input, or probing.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcErrored`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 67 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, rate(rpc_method_errored_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Operations / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 15 + }, + { + "title": "RPC Latency - All Methods", + "description": "###### What this is:\n*Aggregate RPC handler latency across all methods (p75 and p99).*\n\n###### How it's computed:\n*Percentiles of the method-duration histogram over a 5-minute window.*\n\n###### Reading it:\n*p75 reflects typical responsiveness; p99 captures the slow tail.*\n\n###### Healthy range:\n*Sub-millisecond to low-millisecond for light commands; heavier commands run longer.*\n\n###### Watch for:\n*A rising p99 while p75 stays flat \u2014 a subset of calls degrading, often from expensive queries.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.75, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m]))), \"series\", \"p75\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m]))), \"series\", \"p99\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "axisLabel": "Duration (\u03bcs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 16 + }, + { + "title": "Per-Method Latency (p99, Top 10 Slowest)", + "description": "###### What this is:\n*The ten slowest RPC methods by tail latency.*\n\n###### How it's computed:\n*p99 of each method's duration histogram over 5 minutes, top ten.*\n\n###### Reading it:\n*Surfaces which specific methods are expensive.*\n\n###### Healthy range:\n*Method-dependent; ledger/account queries are heavier than status calls.*\n\n###### Watch for:\n*A method whose p99 climbs over time, or an unexpectedly cheap method appearing here.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 83 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, histogram_quantile(0.99, sum by (le, method, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m])))), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Duration (\u03bcs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 17 + }, + { + "title": "RPC Error Ratio by Method", + "description": "###### What this is:\n*The methods with the highest error rates, for spotting failure hotspots.*\n\n###### How it's computed:\n*Per-second error rate over 5 minutes, per method, top ten.*\n\n###### Reading it:\n*Highlights where failures concentrate.*\n\n###### Healthy range:\n*Near zero.*\n\n###### Watch for:\n*One method with a persistently high error rate \u2014 malformed requests or targeted probing.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcErrored`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 91 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, rate(rpc_method_errored_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval]) / (rate(rpc_method_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval]) > 0)), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 5, + "axisLabel": "Error Ratio", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.25 + } + ] + } + }, + "overrides": [] + }, + "id": 18 + }, + { + "title": "Current RPC Latency (p99 Gauge)", + "description": "###### What this is:\n*Current tail latency (p99) of RPC handling across all methods, as a live gauge.*\n\n###### How it's computed:\n*p99 of the method-duration histogram over the recent window.*\n\n###### Reading it:\n*A single at-a-glance number for current RPC responsiveness.*\n\n###### Healthy range:\n*Low-millisecond under normal load.*\n\n###### Watch for:\n*Sustained elevation, indicating the node is under query pressure.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "gauge", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 99 + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (service_instance_id, le, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m]))), \"series\", \"p99 Latency\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100000 + }, + { + "color": "red", + "value": 1000000 + } + ] + } + }, + "overrides": [] + }, + "id": 19 + } + ], + "schemaVersion": 39, + "tags": ["rpc"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "command", + "label": "RPC Command", + "description": "Filter by RPC command name (e.g., server_info, submit)", + "type": "query", + "query": "label_values(span_calls_total{span_name=~\"rpc.command.*\"}, command)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "method", + "label": "RPC Method", + "description": "Filter by RPC method", + "type": "query", + "query": "label_values(rpc_method_started_total, method)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "RPC Performance", + "uid": "rpc-performance", + "refresh": "30s", + "description": "What this shows: Per-command and per-method RPC performance: call rates, error rates, and latency distributions. \u2014 Use it to: Identify slow or failing RPC commands and track client-facing request latency." +} diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json new file mode 100644 index 00000000000..ed0e77c0c6b --- /dev/null +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -0,0 +1,1110 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Transaction Apply Failed Rate", + "description": "###### What this is:\n*Transactions that fail during the apply (transactor) stage, per second \u2014 i.e. the transactor ran but the result was not tesSUCCESS.*\n\n###### How it's computed:\n*Per-second rate of tx.transactor spans at the apply stage whose ter_result is not tesSUCCESS, over the dashboard rate interval, per node.*\n\n###### Reading it:\n*A steady low background is normal (tefPAST_SEQ, tecUNFUNDED, etc.); the transactor legitimately rejects many transactions.*\n\n###### Healthy range:\n*A low, stable rate; workload-dependent.*\n\n###### Watch for:\n*A sharp rise above baseline, which points to a submission pattern that is systematically failing at apply.*\n\n###### Keywords:\n- **Transactor** *(per node)* \u2014 the rippled component that executes one transaction's type-specific logic against the ledger.\n- **Transaction result codes** *(network-wide)* \u2014 the outcome code a transaction returns \u2014 tesSUCCESS, or a tec/tef/tem/ter/tel class code on failure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[Transactor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/tx/detail/Transactor.cpp)\n\n###### Function:\n`Transactor::operator()`\n\n###### References:\n[Transactor](https://xrpl.org/docs/references/protocol/transactions/transaction-results) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transactor)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.transactor\", stage=\"apply\", ter_result!~\"tesSUCCESS|\"}[$__rate_interval])), \"series\", \"Failed / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "id": 1 + }, + { + "title": "Transaction Processing Latency by Type", + "description": "###### What this is:\n*The 95th-percentile time to process a transaction, broken down by transaction type.*\n\n###### How it's computed:\n*95th-percentile of processing durations over 5 minutes, grouped by transaction type and node.*\n\n###### Reading it:\n*Lower is better; compare types to find the expensive ones.*\n\n###### Healthy range:\n*A few milliseconds for simple payments; workload- and type-dependent.*\n\n###### Watch for:\n*A type whose latency climbs, pointing to expensive processing or resource pressure.*\n\n###### Keywords:\n- **Transaction type** *(network-wide)* \u2014 the kind of transaction (Payment, OfferCreate, TrustSet, AMM*, NFToken*, etc.), used as a breakdown dimension.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction`\n\n###### References:\n[Transaction type](https://xrpl.org/docs/references/protocol/transactions/types) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-type)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 4 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max", "lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, tx_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m]))), \"series\", \"$1\", \"tx_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 2 + }, + { + "title": "Transaction Rate by Type", + "description": "###### What this is:\n*How many transactions of each type enter processing per second.*\n\n###### How it's computed:\n*Per-second processing rate over 5 minutes, grouped by transaction type and node.*\n\n###### Reading it:\n*Shows the transaction mix; payments usually lead.*\n\n###### Healthy range:\n*Workload-dependent; mix reflects network activity.*\n\n###### Watch for:\n*A single type spiking far above baseline, consistent with a spam campaign of that type.*\n\n###### Keywords:\n- **Transaction type** *(network-wide)* \u2014 the kind of transaction (Payment, OfferCreate, TrustSet, AMM*, NFToken*, etc.), used as a breakdown dimension.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction`\n\n###### References:\n[Transaction type](https://xrpl.org/docs/references/protocol/transactions/types) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-type)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 12 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (tx_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[$__rate_interval])), \"series\", \"$1\", \"tx_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "TX / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 3 + }, + { + "title": "Transaction Results by Type", + "description": "###### What this is:\n*The non-success result codes transactions return, by transaction type.*\n\n###### How it's computed:\n*Per-second rate of failing transactions over 5 minutes, grouped by type and result code.*\n\n###### Reading it:\n*Fewer failures is better; use it to see which types fail and why.*\n\n###### Healthy range:\n*Workload-dependent; a modest background of expected failures is normal.*\n\n###### Watch for:\n*A surge of one failure code for one type, indicating a systemic issue or abusive submissions.*\n\n###### Keywords:\n- **Transaction result codes** *(network-wide)* \u2014 the outcome code a transaction returns \u2014 tesSUCCESS, or a tec/tef/tem/ter/tel class code on failure.\n- **Transaction type** *(network-wide)* \u2014 the kind of transaction (Payment, OfferCreate, TrustSet, AMM*, NFToken*, etc.), used as a breakdown dimension.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction`\n\n###### References:\n[Transaction result codes](https://xrpl.org/docs/references/protocol/transactions/transaction-results) \u00b7 [Transaction type](https://xrpl.org/docs/references/protocol/transactions/types) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-result-codes)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 20 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (tx_type, ter_result, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.process\", tx_type=~\"$tx_type\", ter_result=~\"$ter_result\", ter_result!=\"tesSUCCESS\"}[$__rate_interval])), \"series\", \"$1\", \"tx_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"ter_result\", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "axisLabel": "Failed TX / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + }, + "overrides": [] + }, + "id": 4 + }, + { + "title": "Transaction Receive vs Suppressed", + "description": "###### What this is:\n*Raw transactions received from peers, split by whether they were duplicates suppressed before processing.*\n\n###### How it's computed:\n*Per-second rate of received transactions over the window, split by suppressed flag and node.*\n\n###### Reading it:\n*A large suppressed share is normal, since the same transaction arrives from many peers.*\n\n###### Healthy range:\n*Workload-dependent; suppressed typically exceeds newly processed.*\n\n###### Watch for:\n*A collapse in suppression (duplicate filtering failing) or an overall receive flood.*\n\n###### Keywords:\n- **Transaction suppression** *(per node)* \u2014 dropping a transaction already seen from another peer, so it is not reprocessed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::handleTransaction`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-suppression)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (suppressed, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{span_name=\"tx.receive\", tx_type=~\"$tx_type\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Suppressed $1\", \"suppressed\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Transactions / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 5 + }, + { + "title": "Transaction Processing Rate", + "description": "###### What this is:\n*Transactions entering the processing pipeline versus raw transactions arriving from peers.*\n\n###### How it's computed:\n*Per-second rates of processed transactions and received peer transactions over 5 minutes, per node.*\n\n###### Reading it:\n*Received sits above processed; the difference is deduplicated relay traffic.*\n\n###### Healthy range:\n*Workload-dependent; both scale with network volume.*\n\n###### Watch for:\n*A receive rate spiking well above processing, consistent with relay flooding.*\n\n###### Keywords:\n- **Transaction suppression** *(per node)* \u2014 dropping a transaction already seen from another peer, so it is not reprocessed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp) \u00b7 [PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction ; PeerImp::handleTransaction`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-suppression)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[$__rate_interval])), \"series\", \"tx.process / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.receive\", tx_type=~\"$tx_type\"}[$__rate_interval])), \"series\", \"tx.receive / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Transactions / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 6 + }, + { + "title": "Transaction Path Distribution", + "description": "###### What this is:\n*The split of transactions by origin: submitted locally versus relayed from peers.*\n\n###### How it's computed:\n*Processed-transaction rate over 5 minutes, split by local-origin flag and shown as proportions.*\n\n###### Reading it:\n*Most traffic on a network node is peer-relayed; local dominates on a submission node.*\n\n###### Healthy range:\n*Workload-dependent on the node's role.*\n\n###### Watch for:\n*An unexpected surge in local submissions, consistent with a local client flooding the node.*\n\n###### Keywords:\n- **Local vs relayed transactions** *(per node)* \u2014 the origin split \u2014 transactions submitted directly to this node versus relayed from peers.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#local-vs-relayed-transactions)", + "type": "piechart", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps" + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (local, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", local=~\"$tx_origin\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[$__rate_interval])), \"series\", \"Local $1\", \"local\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "id": 7 + }, + { + "title": "Transaction Processing Duration Heatmap", + "description": "###### What this is:\n*The full distribution of transaction processing times over the window.*\n\n###### How it's computed:\n*Counts of transactions in each latency band per 5-minute window, shown as color density.*\n\n###### Reading it:\n*A tight low band is healthy; spread bands mean uneven processing time.*\n\n###### Healthy range:\n*Most mass in the low-latency bands; workload-dependent.*\n\n###### Watch for:\n*A distinct slow cluster (multi-modal timing) hidden by percentile charts.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction`", + "type": "heatmap", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "yAxis": { + "axisLabel": "Duration (ms)", + "unit": "ms" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m])) by (le)", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + } + }, + "id": 8 + }, + { + "title": "Transactor Duration by Type (p95)", + "description": "###### What this is:\n*The 95th-percentile execution time of the core transactor step, by transaction type.*\n\n###### How it's computed:\n*95th-percentile of transactor durations over 5 minutes, grouped by type and node.*\n\n###### Reading it:\n*Lower is better; reveals which types are most expensive to execute.*\n\n###### Healthy range:\n*Sub-millisecond to a few ms for most types; workload-dependent.*\n\n###### Watch for:\n*A type whose execution time grows, indicating expensive logic or ledger contention.*\n\n###### Keywords:\n- **Transactor** *(per node)* \u2014 the rippled component that executes one transaction's type-specific logic against the ledger.\n- **Transaction type** *(network-wide)* \u2014 the kind of transaction (Payment, OfferCreate, TrustSet, AMM*, NFToken*, etc.), used as a breakdown dimension.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[Transactor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/Transactor.cpp)\n\n###### Function:\n`Transactor::operator()`\n\n###### References:\n[Transactor](https://xrpl.org/docs/references/protocol/transactions/transaction-results) \u00b7 [Transaction type](https://xrpl.org/docs/references/protocol/transactions/types) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transactor)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 52 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, tx_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.transactor\", tx_type=~\"$tx_type\"}[5m]))), \"series\", \"$1\", \"tx_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 10 + }, + { + "title": "TxQ Accept: Applied Ratio per Node (State Timeline)", + "description": "###### What this is:\n*The applied fraction of TxQ accepts per node over time: the share of queued transactions that were included in a ledger versus removed on failure.*\n\n###### How it's computed:\n*Per node, applied accepts divided by applied-plus-failed accepts over the window.*\n\n###### Reading it:\n*Green (>=90% applied) is a healthy drain; yellow is degraded; red means accepts are mostly failing.*\n\n###### Healthy range:\n*At or near 100% applied when the queue is draining healthily; workload-dependent.*\n\n###### Watch for:\n*A node dropping into yellow or red, which signals queue pressure, under-bidding, or fee escalation on that node.*\n\n###### Keywords:\n- **Transaction queue (TxQ)** *(per node)* \u2014 holds transactions that meet local cost but not the open-ledger cost, to include in a later ledger.\n- **Queue accept (drain)** *(per node)* \u2014 moving queued transactions into a newly closed ledger; the applied share shows drain health.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::accept`\n\n###### References:\n[Transaction queue (TxQ)](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Fee escalation](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-queue-txq)", + "type": "state-timeline", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 60 + }, + "options": { + "mergeValues": true, + "showValue": "auto", + "alignValue": "center", + "rowHeight": 0.85, + "tooltip": { + "mode": "single" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=\"applied\"}[$__rate_interval]))\n/\nsum by (service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=~\"applied|failed\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "{{service_instance_id}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "green", + "value": 0.9 + } + ] + }, + "custom": { + "fillOpacity": 80, + "lineWidth": 0 + } + }, + "overrides": [] + }, + "id": 11 + }, + { + "title": "Tx Apply Pipeline Latency by Stage (p95)", + "description": "###### What this is:\n*The 95th-percentile duration of each apply-pipeline stage.*\n\n###### How it's computed:\n*95th-percentile of per-stage durations over 5 minutes, grouped by stage and node.*\n\n###### Reading it:\n*Lower is better; isolates which stage dominates processing time.*\n\n###### Healthy range:\n*Each stage typically sub-millisecond to a few ms; workload-dependent.*\n\n###### Watch for:\n*One stage's latency rising, pinpointing the bottleneck in transaction handling.*\n\n###### Keywords:\n- **Apply pipeline stages** *(per node)* \u2014 the ordered checks a transaction passes \u2014 preflight (stateless), preclaim (stateful), then apply.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[applySteps.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/applySteps.cpp) \u00b7 [Transactor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/Transactor.cpp)\n\n###### Function:\n`makeStageSpan ; Transactor::operator()`\n\n###### References:\n[Apply pipeline stages](https://xrpl.org/docs/concepts/transactions) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#apply-pipeline-stages)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 68 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(label_replace(label_replace(label_replace(histogram_quantile(0.95, sum by (le, stage, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", stage=~\"$stage\"}[5m]))), \"stage\", \"Preflight\", \"stage\", \"preflight\"), \"stage\", \"Preclaim\", \"stage\", \"preclaim\"), \"stage\", \"Apply\", \"stage\", \"apply\"), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 12 + }, + { + "title": "Tx Apply Pipeline Failure Rate by Stage", + "description": "###### What this is:\n*How many transactions fail at each apply-pipeline stage per second.*\n\n###### How it's computed:\n*Per-second rate of non-success outcomes over 5 minutes, grouped by stage and node.*\n\n###### Reading it:\n*Shows whether rejections concentrate in preflight, preclaim, or apply.*\n\n###### Healthy range:\n*Workload-dependent; a modest background of expected rejections is normal.*\n\n###### Watch for:\n*A failure spike concentrated in one stage, consistent with malformed or spam submissions.*\n\n###### Keywords:\n- **Apply pipeline stages** *(per node)* \u2014 the ordered checks a transaction passes \u2014 preflight (stateless), preclaim (stateful), then apply.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[applySteps.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/applySteps.cpp) \u00b7 [Transactor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/Transactor.cpp)\n\n###### Function:\n`makeStageSpan ; Transactor::operator()`\n\n###### References:\n[Apply pipeline stages](https://xrpl.org/docs/concepts/transactions) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#apply-pipeline-stages)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 76 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(label_replace(label_replace(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", stage=~\"$stage\", ter_result!~\"tesSUCCESS|\"}[$__rate_interval])), \"stage\", \"Preflight\", \"stage\", \"preflight\"), \"stage\", \"Preclaim\", \"stage\", \"preclaim\"), \"stage\", \"Apply\", \"stage\", \"apply\"), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Failed Spans / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 13 + }, + { + "title": "Tx Apply Pipeline Latency by Type and Stage (p95)", + "description": "###### What this is:\n*The 95th-percentile stage duration broken down by both transaction type and pipeline stage.*\n\n###### How it's computed:\n*95th-percentile durations over 5 minutes, grouped by type and stage; higher cardinality than the by-stage view.*\n\n###### Reading it:\n*Lower is better; shows which stage dominates each type's latency.*\n\n###### Healthy range:\n*Workload-dependent; most type/stage pairs sub-millisecond to a few ms.*\n\n###### Watch for:\n*A specific type/stage combination rising sharply, pinpointing a costly path.*\n\n###### Keywords:\n- **Transaction type** *(network-wide)* \u2014 the kind of transaction (Payment, OfferCreate, TrustSet, AMM*, NFToken*, etc.), used as a breakdown dimension.\n- **Apply pipeline stages** *(per node)* \u2014 the ordered checks a transaction passes \u2014 preflight (stateless), preclaim (stateful), then apply.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[applySteps.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/applySteps.cpp) \u00b7 [Transactor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/tx/Transactor.cpp)\n\n###### Function:\n`makeStageSpan ; Transactor::operator()`\n\n###### References:\n[Transaction type](https://xrpl.org/docs/references/protocol/transactions/types) \u00b7 [Apply pipeline stages](https://xrpl.org/docs/concepts/transactions) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-type)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 84 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(label_replace(label_replace(label_replace(histogram_quantile(0.95, sum by (le, tx_type, stage, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", tx_type=~\"$tx_type\", stage=~\"$stage\"}[5m]))), \"stage\", \"Preflight\", \"stage\", \"preflight\"), \"stage\", \"Preclaim\", \"stage\", \"preclaim\"), \"stage\", \"Apply\", \"stage\", \"apply\"), \"series\", \"$1\", \"tx_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"stage\", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + }, + "overrides": [] + }, + "id": 14 + }, + { + "title": "Transaction Apply Duration per Ledger", + "description": "###### What this is:\n*The 95th-percentile time to apply the agreed transaction set into each new ledger.*\n\n###### How it's computed:\n*95th-percentile of transaction-apply durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; a major component of ledger build time.*\n\n###### Healthy range:\n*A few to tens of milliseconds; scales with transactions per ledger.*\n\n###### Watch for:\n*Rising durations during heavy or expensive transaction sets.*\n\n###### Keywords:\n- **Transaction apply phase** *(per node)* \u2014 the step that executes the agreed transaction set into the new ledger during a close.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **In-ledger vs target count** *(per node)* \u2014 transactions already in the open ledger versus the soft target that triggers fee escalation.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`applyTransactions`\n\n###### References:\n[Transaction apply phase](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [In-ledger vs target count](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-apply-phase)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 92 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.apply\"}[5m]))), \"series\", \"tx.apply\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 15 + }, + { + "title": "TxQ Enqueue Rate by Transaction Type", + "description": "###### What this is:\n*How many transactions are queued (rather than applied immediately) per second, by type.*\n\n###### How it's computed:\n*Per-second rate of queue-enqueue operations over 5 minutes, grouped by transaction type and node.*\n\n###### Reading it:\n*Shows the demand mix hitting the queue and how it shifts as the queue fills.*\n\n###### Healthy range:\n*Workload-dependent; low or zero when the network is not congested.*\n\n###### Watch for:\n*A spam burst of one type, an early indicator of fee escalation.*\n\n###### Keywords:\n- **Direct apply vs enqueue** *(per node)* \u2014 whether a transaction applied straight to the open ledger or was placed in the queue for a later ledger.\n- **Transaction type** *(network-wide)* \u2014 the kind of transaction (Payment, OfferCreate, TrustSet, AMM*, NFToken*, etc.), used as a breakdown dimension.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::apply`\n\n###### References:\n[Direct apply vs enqueue](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Transaction type](https://xrpl.org/docs/references/protocol/transactions/types) \u00b7 [Fee escalation](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#direct-apply-vs-enqueue)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 100 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (tx_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.enqueue\"}[$__rate_interval])), \"series\", \"$1\", \"tx_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Enqueues / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 16 + }, + { + "title": "Queue Bypass Ratio (Direct Apply vs Enqueue)", + "description": "###### What this is:\n*The fraction of transactions that applied straight to the open ledger instead of being queued.*\n\n###### How it's computed:\n*Direct-apply rate divided by direct-apply plus enqueue rate over 5 minutes, per node.*\n\n###### Reading it:\n*Higher is better; near 1.0 means the network is not congested.*\n\n###### Healthy range:\n*Close to 1.0 when not congested; falls as demand exceeds capacity.*\n\n###### Watch for:\n*A falling ratio, the cleanest single signal the network has entered sustained fee escalation.*\n\n###### Keywords:\n- **Direct apply vs enqueue** *(per node)* \u2014 whether a transaction applied straight to the open ledger or was placed in the queue for a later ledger.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::tryDirectApply ; TxQ::apply`\n\n###### References:\n[Direct apply vs enqueue](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Fee escalation](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#direct-apply-vs-enqueue)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 108 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.apply_direct\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.apply_direct\"}[$__rate_interval])) + sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.enqueue\"}[$__rate_interval])), 1), \"series\", \"Direct-Apply Fraction\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percentunit", + "custom": { + "axisLabel": "Bypass Fraction", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 18 + }, + { + "title": "Queue Accept (Drain) Duration per Ledger", + "description": "###### What this is:\n*The 95th-percentile time to drain queued transactions into a newly closed ledger.*\n\n###### How it's computed:\n*95th-percentile of queue-drain durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; rising time signals queue pressure at ledger close.*\n\n###### Healthy range:\n*A few milliseconds when the queue is light; workload-dependent.*\n\n###### Watch for:\n*A sustained climb, indicating a large or contended queue slowing ledger close.*\n\n###### Keywords:\n- **Queue accept (drain)** *(per node)* \u2014 moving queued transactions into a newly closed ledger; the applied share shows drain health.\n- **Ledger close** *(network event)* \u2014 the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::accept`\n\n###### References:\n[Queue accept (drain)](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#queue-accept-drain)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 108 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept\"}[5m]))), \"series\", \"Drain\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 19 + }, + { + "title": "Queue Cleanup Rate (Expired Entries)", + "description": "###### What this is:\n*How often expired transactions are removed from the queue each ledger, per second.*\n\n###### How it's computed:\n*Per-second rate of queue-cleanup operations over 5 minutes, per node.*\n\n###### Reading it:\n*Low is healthy; a rising rate means submitters are abandoning transactions.*\n\n###### Healthy range:\n*Near zero when not congested; workload-dependent.*\n\n###### Watch for:\n*A rising rate, a demand-frustration signal where submitters under-bid the escalating fee and gave up.*\n\n###### Keywords:\n- **Queue expiry / abandonment** *(per node)* \u2014 removing a queued transaction whose LastLedgerSequence deadline passed before inclusion.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::processClosedLedger`\n\n###### References:\n[Queue expiry / abandonment](https://xrpl.org/docs/concepts/transactions/reliable-transaction-submission) \u00b7 [Fee escalation](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#queue-expiry-abandonment)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 116 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.cleanup\"}[$__rate_interval])), \"series\", \"Cleanups / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ops", + "custom": { + "axisLabel": "Cleanups / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 20 + } + ], + "schemaVersion": 39, + "tags": ["transactions"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "tx_origin", + "label": "TX Origin", + "description": "Filter by transaction origin (true = local submit, false = peer relay)", + "type": "query", + "query": "label_values(span_calls_total{span_name=\"tx.process\"}, local)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "tx_type", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(span_calls_total{span_name=\"tx.process\", tx_type!=\"\"}, tx_type)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "label": "TX Type" + }, + { + "name": "ter_result", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(span_calls_total{span_name=\"tx.process\", ter_result!=\"\"}, ter_result)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "label": "Result Code" + }, + { + "name": "stage", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(span_calls_total{span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", stage!=\"\"}, stage)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "label": "Apply Stage" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Transaction Overview", + "uid": "transaction-overview", + "refresh": "30s", + "description": "What this shows: Transaction flow through this node: receipt, processing, results, per-stage timing, and queue behavior. \u2014 Use it to: Trace transactions from arrival to ledger, and locate stalls in processing or the queue." +} diff --git a/docker/telemetry/grafana/dashboards/validate_dashboards.py b/docker/telemetry/grafana/dashboards/validate_dashboards.py new file mode 100755 index 00000000000..e921d02c4f2 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/validate_dashboards.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Dashboard lint: cumulative metrics must be rate()-wrapped; tier filters present.""" + +import json, re, sys + +# Prometheus gauges that hold a CUMULATIVE total -> must be rate()/increase()-wrapped. +CUMULATIVE_PREFIXES = ( + "total_bytes_", + "total_messages_", + "transactions_messages_", + "transactions_duplicate_messages_", + "proposals_", + "validations_", + "overlay_peer_disconnects", + "squelch_", + "overhead_", + "validator_lists_", + "set_get_", + "set_share_", + "have_transactions_", + "requested_transactions_", + "proof_path_", + "replay_delta_", + "ledger_data_", + "ledger_get_", + "ledger_share_", + "ledger_transaction_", + "ledger_account_state_", + "getobject_", + "jq_trans_overflow_total", +) +# nodestore_state{metric=...} cumulative sub-series (raw only inside these metric= labels). +NODESTORE_CUMULATIVE = ( + "node_reads_total", + "node_reads_hit", + "node_writes", + "node_read_bytes", + "node_written_bytes", + "node_reads_duration_us", +) +# state_accounting_*_duration are cumulative µs. +STATE_DURATION = re.compile(r"state_accounting_\w+_duration") +REQUIRED_FILTERS = ( + "$node", + "$deployment_environment", + "$xrpl_network_type", + "$service_name", +) + + +def iter_panels(dash): + for p in dash.get("panels", []): + yield p + for sub in p.get("panels", []) or []: + yield sub + + +def expr_is_wrapped(expr): + return ( + "rate(" in expr + or "increase(" in expr + or "irate(" in expr + or "histogram_quantile(" in expr + ) + + +def check(path, forbid_5m): + errs = [] + try: + dash = json.load(open(path)) + except Exception as e: + return [f"{path}: INVALID JSON: {e}"] + for p in iter_panels(dash): + title = p.get("title", "") + for tg in p.get("targets", []) or []: + expr = (tg.get("expr") or "").strip() + if not expr: + continue + # tier filters + if "{" in expr: + for f in REQUIRED_FILTERS: + if f not in expr: + errs.append(f"{path} [{title}]: missing {f} in expr") + # cumulative metric plotted raw? + hits = [m for m in CUMULATIVE_PREFIXES if m in expr] + nod = [m for m in NODESTORE_CUMULATIVE if f'"{m}"' in expr] + statedur = STATE_DURATION.search(expr) + if (hits or nod or statedur) and not expr_is_wrapped(expr): + who = hits or nod or [statedur.group(0)] + errs.append( + f"{path} [{title}]: cumulative metric {who} plotted RAW (needs rate/increase)" + ) + if "deriv(" in expr or "idelta(" in expr: + errs.append( + f"{path} [{title}]: uses deriv()/idelta() on a cumulative series" + ) + if forbid_5m and ("[5m]" in expr or "[1h]" in expr): + # Only a fixed [5m]/[1h] inside a top-level rate()/increase()/irate() on a + # plain counter is a violation. Legitimate fixed-window exprs -- histogram + # buckets (rate(..._bucket[5m])), avg_over_time windows, and subqueries + # ([5m:]) -- are intentional and must not be flagged. + if not ("_bucket" in expr or "avg_over_time" in expr or ":]" in expr): + errs.append( + f"{path} [{title}]: hardcoded range window; use [$__rate_interval]" + ) + # unit check + unit = p.get("fieldConfig", {}).get("defaults", {}).get("unit", "") + if unit == "mps": + errs.append(f"{path} [{title}]: invalid Grafana unit 'mps' (use 'cps')") + return errs + + +def main(): + args = [a for a in sys.argv[1:] if not a.startswith("--")] + forbid_5m = "--no-5m" in sys.argv + all_errs = [] + for path in args: + all_errs += check(path, forbid_5m) + if all_errs: + print("\n".join(all_errs)) + print(f"\nFAIL: {len(all_errs)} violation(s)") + sys.exit(1) + print(f"OK: {len(args)} dashboard(s) passed") + + +if __name__ == "__main__": + main() diff --git a/docker/telemetry/grafana/dashboards/validator-health.json b/docker/telemetry/grafana/dashboards/validator-health.json new file mode 100644 index 00000000000..aafb44e99cc --- /dev/null +++ b/docker/telemetry/grafana/dashboards/validator-health.json @@ -0,0 +1,1148 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Validator agreement rates, validation counts, amendment and UNL status, server state tracking, and ledger close rate. \u2014 Use it to: Confirm the validator is participating, agreeing with the network, and closing ledgers at the expected cadence.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "Validation Agreement", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [] + }, + { + "title": "Agreement % (1h)", + "description": "###### What this is:\n*Share of ledgers over the last hour where this validator agreed with the network consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 1-hour agreement percentage.*\n\n###### Reading it:\n*Higher is better; green at 95%+, yellow from 80%, red below.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*Values below 80%, meaning the validator frequently disagrees with consensus.*\n\n###### Keywords:\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 1 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"agreement_pct_1h\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Agreement 1h\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "green", + "value": 95 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Agreement % (24h)", + "description": "###### What this is:\n*Share of ledgers over the last 24 hours where this validator agreed with consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 24-hour agreement percentage.*\n\n###### Reading it:\n*Higher is better; a smoother, longer-term view than the 1h stat.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*A sustained dip below 90%, which can indicate configuration drift or a network partition.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 9 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"agreement_pct_24h\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Agreement 24h\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "green", + "value": 95 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Agreements vs Missed (1h)", + "description": "###### What this is:\n*Counts of agreed versus missed validations over the last hour.*\n\n###### How it's computed:\n*Instantaneous gauge readings of the 1-hour agreed and missed counts.*\n\n###### Reading it:\n*Agreements should dominate; missed should be small.*\n\n###### Healthy range:\n*Missed near 0.*\n\n###### Watch for:\n*A high missed count, meaning the validator is skipping consensus rounds.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 17 + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"agreements_1h\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Agreements 1h\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"missed_1h\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Missed 1h\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "Missed.*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + } + }, + { + "title": "Agreements vs Missed (24h)", + "description": "###### What this is:\n*Counts of agreed versus missed validations over the last 24 hours.*\n\n###### How it's computed:\n*Instantaneous gauge readings of the 24-hour agreed and missed counts.*\n\n###### Reading it:\n*Agreements should dominate over the full day.*\n\n###### Healthy range:\n*Missed a small fraction of agreements.*\n\n###### Watch for:\n*A growing missed share, signalling longer-term reliability problems.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"agreements_24h\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Agreements 24h\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"missed_24h\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Missed 24h\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "Missed.*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + } + }, + { + "title": "Validation Rates", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "collapsed": false, + "panels": [] + }, + { + "title": "Validation Rate", + "description": "###### What this is:\n*Validations this node sends per minute.*\n\n###### How it's computed:\n*Per-second rate of the sent-validations counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Should track the ledger close cadence; roughly one validation per closed ledger.*\n\n###### Healthy range:\n*About 12-20 per minute (one per closed ledger, ~3-5s close).*\n\n###### Watch for:\n*A drop toward zero, meaning the validator has stopped participating.*\n\n###### Keywords:\n- **Validations checked vs sent** *(per node)* \u2014 validations this node received from peers and checked, versus validations it issued itself.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementValidationsSent (caller RCLConsensus.cpp)`\n\n###### References:\n[Validations checked vs sent](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validations-checked-vs-sent)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_sent_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Sent/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cpm", + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "green", + "value": 8 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Validations Checked Rate", + "description": "###### What this is:\n*Validations received from peers and checked per minute.*\n\n###### How it's computed:\n*Per-second rate of the checked-validations counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Reflects how much validation traffic the network is delivering to this node.*\n\n###### Healthy range:\n*Workload-dependent; scales with trusted validator count.*\n\n###### Watch for:\n*A sudden collapse, which suggests peer connectivity loss or network isolation.*\n\n###### Keywords:\n- **Validations checked vs sent** *(per node)* \u2014 validations this node received from peers and checked, versus validations it issued itself.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementValidationsChecked (caller NetworkOPs.cpp)`\n\n###### References:\n[Validations checked vs sent](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validations-checked-vs-sent)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 42 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validations_checked_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Checked/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cpm", + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Amendment Blocked", + "description": "###### What this is:\n*Whether the node is amendment-blocked (OK/BLOCKED).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the amendment-blocked flag.*\n\n###### Reading it:\n*OK is healthy; BLOCKED means an enabled amendment is unsupported by this build.*\n\n###### Healthy range:\n*OK.*\n\n###### Watch for:\n*A BLOCKED state, which halts validation and requires a software upgrade.*\n\n###### Keywords:\n- **Amendment blocked** *(per node)* \u2014 the node has halted because the network enabled an amendment its software version does not support.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[Amendment blocked](https://xrpl.org/docs/concepts/networks-and-servers/amendments) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#amendment-blocked)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 50 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validator_health{metric=\"amendment_blocked\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Blocked\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "OK", + "color": "green" + } + } + }, + { + "type": "value", + "options": { + "1": { + "text": "BLOCKED", + "color": "red" + } + } + } + ], + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "UNL Expiry (days)", + "description": "###### What this is:\n*Days remaining until the current UNL (trusted validator list) expires.*\n\n###### How it's computed:\n*Instantaneous gauge reading of days-to-expiry.*\n\n###### Reading it:\n*Higher is safer; green at 30+, yellow under 7, red at expiry.*\n\n###### Healthy range:\n*30+ days.*\n\n###### Watch for:\n*Fewer than 7 days, after which the node loses its trusted validator set if not renewed.*\n\n###### Keywords:\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-unique-node-list)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 58 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validator_health{metric=\"unl_expiry_days\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"UNL Expiry\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "d", + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 7 + }, + { + "color": "green", + "value": 30 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "UNL Blocked", + "description": "###### What this is:\n*Whether the node's UNL is blocked (OK/BLOCKED).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the UNL-blocked flag.*\n\n###### Reading it:\n*OK is healthy; BLOCKED means validator trust cannot be established.*\n\n###### Healthy range:\n*OK.*\n\n###### Watch for:\n*A BLOCKED state, which can stop the node participating in consensus.*\n\n###### Keywords:\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n- **UNL blocked** *(per node)* \u2014 the node cannot establish a usable trusted validator list, so it cannot safely validate.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [UNL blocked](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-unique-node-list)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 66 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validator_health{metric=\"unl_blocked\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"UNL Blocked\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "OK", + "color": "green" + } + } + }, + { + "type": "value", + "options": { + "1": { + "text": "BLOCKED", + "color": "red" + } + } + } + ], + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Agreement/Missed Counters (Rate)", + "description": "###### What this is:\n*Rate of cumulative agreement and missed-validation counters per minute.*\n\n###### How it's computed:\n*Per-second rate of each monotonic counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Agreements should dominate; the missed line should stay low.*\n\n###### Healthy range:\n*Missed rate near 0.*\n\n###### Watch for:\n*A rising missed rate, complementing the windowed agreement percentages above.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationTotalsCounters`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 74 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validation_agreements_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Agreements/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(validation_missed_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Missed/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cpm", + "custom": { + "axisLabel": "Per Minute", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "Missed.*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + } + }, + { + "title": "Server State & Consensus", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 82 + }, + "collapsed": false, + "panels": [] + }, + { + "title": "Validation Quorum", + "description": "###### What this is:\n*Minimum number of trusted validations required to declare a ledger fully validated.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the current quorum requirement.*\n\n###### Reading it:\n*Tracks the quorum derived from the active validator list; changes when the list changes.*\n\n###### Healthy range:\n*Stable at the network-appropriate value.*\n\n###### Watch for:\n*An unexpected drop, which can weaken consensus safety guarantees.*\n\n###### Keywords:\n- **Validation quorum** *(network-wide)* \u2014 the minimum number of agreeing trusted validations needed to declare a ledger fully validated.\n- **Validator list** *(network-wide)* \u2014 signed lists of recommended validators (UNLs) that peers distribute to each other.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[Validation quorum](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Validator list](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-quorum)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 83 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validator_health{metric=\"validation_quorum\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Quorum\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "none", + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "State Value Timeline", + "description": "###### What this is:\n*Numeric encoding of the server operating state (disconnected, connected, syncing, tracking, full, validating, proposing) over time.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the encoded state value.*\n\n###### Reading it:\n*A flat line at the full-operation state is healthy; steps show transitions.*\n\n###### Healthy range:\n*Steady at the highest (full) state.*\n\n###### Watch for:\n*Frequent transitions, useful for correlating state flapping with other metrics.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n- **Consensus mode** *(per node)* \u2014 the node's role/health in the current round: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStateTrackingGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Consensus mode](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 91 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_tracking{metric=\"state_value\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"State\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "State", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "Time in Current State", + "description": "###### What this is:\n*How long the server has held its current operating state, in seconds.*\n\n###### How it's computed:\n*Current value of the time-in-state gauge.*\n\n###### Reading it:\n*Not yet wired in the code; the value currently always reads 0.*\n\n###### Healthy range:\n*Not applicable; the value is always 0 today.*\n\n###### Watch for:\n*n/a until implemented.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStateTrackingGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 99 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_tracking{metric=\"time_in_current_state_seconds\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Time in State\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "s", + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "State Changes Rate", + "description": "###### What this is:\n*Rate of server operating-state changes per hour.*\n\n###### How it's computed:\n*Per-hour rate of the state-change counter, averaged over a 1-hour window.*\n\n###### Reading it:\n*Near zero is healthy; each increment is one state transition.*\n\n###### Healthy range:\n*Near 0 changes per hour.*\n\n###### Watch for:\n*Frequent transitions, which point to network instability or configuration problems.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementStateChanges (caller NetworkOPs.cpp)`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 107 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(3600 * sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(state_changes_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Changes/hr\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "suffix:/hr", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Ledgers Closed Rate", + "description": "###### What this is:\n*Ledgers closed per minute by this node.*\n\n###### How it's computed:\n*Per-second rate of the ledgers-closed counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Should match the network's steady close cadence.*\n\n###### Healthy range:\n*About 12-20 per minute (one per closed ledger, ~3-5s close).*\n\n###### Watch for:\n*Deviation from the expected cadence, which indicates consensus timing trouble or the node falling behind.*\n\n###### Keywords:\n- **Ledgers closed rate** *(per node)* \u2014 how many ledgers this node closed per minute; should match the network close cadence.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementLedgersClosed (caller RCLConsensus.cpp)`\n\n###### References:\n[Ledgers closed rate](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledgers-closed-rate)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 115 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(ledgers_closed_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Closed/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cpm", + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "green", + "value": 8 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Agreement % (7d)", + "description": "###### What this is:\n*Share of ledgers over the trailing 7 days where this validator agreed with consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 7-day agreement percentage.*\n\n###### Reading it:\n*The long-term reliability window; higher is better.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*A gradual decline, which reflects chronic rather than transient disagreement.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", + "type": "stat", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 123 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"agreement_pct_7d\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Agreement 7d\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "green", + "value": 95 + } + ] + }, + "custom": {} + }, + "overrides": [] + } + }, + { + "title": "Agreements vs Missed (7d)", + "description": "###### What this is:\n*Agreed versus missed validation counts over the trailing 7 days.*\n\n###### How it's computed:\n*Instantaneous gauge readings of the 7-day agreed and missed counts.*\n\n###### Reading it:\n*Agreements should dominate across the week.*\n\n###### Healthy range:\n*Missed a small fraction of agreements.*\n\n###### Watch for:\n*A rising missed trend, signalling sustained validator unreliability.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 131 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"agreements_7d\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Agreements 7d\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(validation_agreement{metric=\"missed_7d\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Missed 7d\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Count", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + } + ], + "schemaVersion": 39, + "tags": ["validator", "health"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by rippled node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Validator Health", + "uid": "validator-health", + "refresh": "30s" +} diff --git a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml new file mode 100644 index 00000000000..922caed0810 --- /dev/null +++ b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml @@ -0,0 +1,65 @@ +# Grafana contact-point provisioning for rippled OTel alerts. +# +# Phase 9: Internal metric gap fill — alerting on health-critical metrics. +# +# A contact point is where a firing alert is delivered. Two are defined: +# xrpld-default — Slack only; receives warning-severity alerts. +# xrpld-critical — Slack + email; receives critical-severity alerts. +# The severity split is wired in policies.yaml. +# +# Secrets and personal addresses are NOT hard-coded. To enable delivery, +# replace the disabled placeholder values below with a real Slack webhook +# and alert email — e.g. by copying docker/telemetry/.env.alerting.example +# to .env.alerting (gitignored) and swapping the placeholder lines to +# ${SLACK_WEBHOOK_URL} / ${ALERT_EMAIL_TO}. See the Alerting section of +# docs/telemetry-runbook.md for setup. +# +# The defaults are deliberately non-empty, invalid-but-valid-shaped +# placeholders (an unroutable webhook host and a .invalid email). Grafana's +# alerting provisioning validator REQUIRES a non-empty Slack url and email +# addresses, and it does NOT support ${VAR:-default} expansion — an unset +# ${VAR} expands to empty and crashes Grafana on startup. The placeholders +# keep the whole stack booting with zero configuration; alerts simply route +# nowhere until a real destination is supplied. +# +# Email delivery additionally requires SMTP configured on the Grafana +# service (GF_SMTP_* in docker-compose.yml). + +apiVersion: 1 + +contactPoints: + # --- Warning tier: Slack only --- + - orgId: 1 + name: xrpld-default + receivers: + - uid: xrpld-slack-default + type: slack + settings: + # Disabled placeholder: an unroutable webhook host. A non-empty url + # selects Slack webhook mode (no recipient/token required) and keeps + # provisioning valid. Replace with a real ${SLACK_WEBHOOK_URL} to + # enable delivery. + url: https://hooks.slack.invalid/disabled + title: "{{ .CommonLabels.alertname }} on {{ .CommonLabels.service_instance_id }}" + disableResolveMessage: false + + # --- Critical tier: Slack + email --- + - orgId: 1 + name: xrpld-critical + receivers: + - uid: xrpld-slack-critical + type: slack + settings: + # Disabled placeholder — see xrpld-slack-default above. + url: https://hooks.slack.invalid/disabled + title: "[CRITICAL] {{ .CommonLabels.alertname }} on {{ .CommonLabels.service_instance_id }}" + disableResolveMessage: false + - uid: xrpld-email-critical + type: email + settings: + # Disabled placeholder: a .invalid address keeps the required + # `addresses` field non-empty so provisioning validates. Replace + # with a real ${ALERT_EMAIL_TO} (and enable SMTP) to deliver. + addresses: alerts-disabled@xrpld.invalid + singleEmail: true + disableResolveMessage: false diff --git a/docker/telemetry/grafana/provisioning/alerting/policies.yaml b/docker/telemetry/grafana/provisioning/alerting/policies.yaml new file mode 100644 index 00000000000..b1c4c92c705 --- /dev/null +++ b/docker/telemetry/grafana/provisioning/alerting/policies.yaml @@ -0,0 +1,44 @@ +# Grafana notification-policy provisioning for rippled OTel alerts. +# +# Phase 9: Internal metric gap fill — alerting on health-critical metrics. +# +# The notification policy tree decides which contact point receives a firing +# alert and how alerts are batched. Routing is split by severity: +# - default (root) → xrpld-default (Slack) — catches warnings. +# - severity = critical → xrpld-critical (Slack + email) — child route. +# A critical alert matches the child route and is delivered to Slack AND email; +# everything else falls through to the root receiver. +# +# Grouping by alertname + service_instance_id means one notification per +# (alert, node) pair, so a fleet-wide problem does not collapse into a single +# ambiguous message. + +apiVersion: 1 + +policies: + - orgId: 1 + receiver: xrpld-default + group_by: + - alertname + - service_instance_id + # Wait before sending the first notification for a new group, so a burst of + # related series is batched together. + group_wait: 30s + # Wait before adding new alerts to an already-notified group. + group_interval: 5m + # Minimum time before re-sending a notification for an alert that is still + # firing. + repeat_interval: 4h + routes: + # Critical alerts page harder: Slack + email. matchers uses the + # PromQL-style label match "severity = critical" set on the rules. + - receiver: xrpld-critical + matchers: + - severity = critical + group_by: + - alertname + - service_instance_id + group_wait: 30s + group_interval: 5m + # Re-page critical alerts more often than the 4h default while still firing. + repeat_interval: 1h diff --git a/docker/telemetry/grafana/provisioning/alerting/rules.yaml b/docker/telemetry/grafana/provisioning/alerting/rules.yaml new file mode 100644 index 00000000000..cf396da0c50 --- /dev/null +++ b/docker/telemetry/grafana/provisioning/alerting/rules.yaml @@ -0,0 +1,410 @@ +# Grafana alert-rule provisioning for rippled OTel metrics. +# +# Phase 9: Internal metric gap fill — alerting on health-critical metrics. +# +# Six rules across three node-fatal subsystems: consensus/ledger health, +# validator health, and the job queue. Every metric referenced here is +# introduced by phase 9's MetricsRegistry. +# +# Rule shape (Grafana server-side evaluation): +# A Prometheus query — a 5-minute rate / histogram_quantile, aggregated +# `by (service_instance_id)` so each node evaluates +# independently. Alert rules run headless and cannot +# use the dashboards' `$node` template variable. +# B reduce (last) — collapse A's series to its most recent value. +# C threshold — the firing condition; `condition: C`. +# +# Thresholds are documented in docs/telemetry-runbook.md (Alerting section) +# and are intended to be tuned. + +apiVersion: 1 + +groups: + # ------------------------------------------------------------------ # + # Consensus / ledger health # + # ------------------------------------------------------------------ # + - orgId: 1 + name: xrpld-consensus + folder: xrpld + interval: 1m + rules: + # A closed ledger that later fails validation against the network — + # any sustained nonzero rate means this node built history the rest + # of the network rejects. + - uid: xrpld-ledger-history-mismatch + title: LedgerHistoryMismatch + condition: C + for: 5m + noDataState: NoData + execErrState: Error + labels: + severity: critical + category: consensus + annotations: + summary: "Ledger history mismatch on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} is recording ledger history + mismatches ({{ $values.B.Value }}/s over 5m). The node's built ledger + diverges from the validated network chain. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_instance_id) (rate(ledger_history_mismatch_total[5m])) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [0] + datasource: + type: __expr__ + uid: __expr__ + + # Healthy nodes close a ledger every ~3-5s. Zero closes for 3 minutes + # means consensus/ledger advancement is stuck. NoData (metric absent) + # also fires — a vanished series here means the node is down. + - uid: xrpld-ledger-close-stalled + title: LedgerCloseStalled + condition: C + for: 3m + noDataState: Alerting + execErrState: Error + labels: + severity: critical + category: consensus + annotations: + summary: "Ledger closing stalled on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} has closed no ledgers for + several minutes (5m rate has decayed to zero). Consensus or ledger + advancement is stuck. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_instance_id) (rate(ledgers_closed_total[5m])) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: lt + params: [0.001] + datasource: + type: __expr__ + uid: __expr__ + + # ------------------------------------------------------------------ # + # Validator health # + # ------------------------------------------------------------------ # + - orgId: 1 + name: xrpld-validator + folder: xrpld + interval: 1m + rules: + # This validator's own validations are not reaching / agreeing with + # the network. A sustained nonzero miss rate risks the validator being + # dropped from UNLs. + - uid: xrpld-validations-missed + title: ValidationsMissed + condition: C + for: 5m + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: validator + annotations: + summary: "Validations missed on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} is missing validations + ({{ $values.B.Value }}/s over 5m). Its validations are not agreeing with + the validated ledger. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_instance_id) (rate(validation_missed_total[5m])) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [0] + datasource: + type: __expr__ + uid: __expr__ + + # The node has stopped checking incoming validations. Zero checked + # validations means it is no longer processing the validation stream + # from peers. + - uid: xrpld-validations-not-checked + title: ValidationsNotChecked + condition: C + for: 5m + noDataState: Alerting + execErrState: Error + labels: + severity: warning + category: validator + annotations: + summary: "No validations checked on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} has checked no incoming + validations for several minutes (5m rate has decayed to zero). The + validation stream from peers may have stopped. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_instance_id) (rate(validations_checked_total[5m])) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: lt + params: [0.001] + datasource: + type: __expr__ + uid: __expr__ + + # ------------------------------------------------------------------ # + # Job queue / resource health # + # ------------------------------------------------------------------ # + - orgId: 1 + name: xrpld-jobqueue + folder: xrpld + interval: 1m + rules: + # Transactions are being dropped because the job queue is full — the + # node is shedding load it cannot process. + - uid: xrpld-jobqueue-tx-overflow + title: JobQueueTxOverflow + condition: C + for: 5m + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: jobqueue + annotations: + summary: "Job queue transaction overflow on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} is overflowing its transaction + job queue ({{ $values.B.Value }}/s over 5m). Transactions are being + dropped under load. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_instance_id) (rate(jq_trans_overflow_total[5m])) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [0] + datasource: + type: __expr__ + uid: __expr__ + + # p99 time a job waits in the queue before running. A sustained p99 + # above 1s means the node is saturated and work is backing up. Tune the + # threshold (in microseconds) to the deployment. + - uid: xrpld-jobqueue-latency-high + title: JobQueueLatencyHigh + condition: C + for: 5m + noDataState: NoData + execErrState: Error + labels: + severity: warning + category: jobqueue + annotations: + summary: "Job queue latency high on {{ $labels.service_instance_id }}" + description: >- + Node {{ $labels.service_instance_id }} has a p99 job-queue wait of + {{ $values.B.Value }}µs (>1s) over 5m. The node is saturated and jobs are + backing up. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + refId: A + expr: histogram_quantile(0.99, sum by (le, service_instance_id) (rate(job_queued_us_bucket[5m]))) + instant: true + range: false + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [1000000] + datasource: + type: __expr__ + uid: __expr__ diff --git a/docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml b/docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 00000000000..dec8dc87c0f --- /dev/null +++ b/docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: xrpld-telemetry + orgId: 1 + folder: xrpld + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/docker/telemetry/grafana/provisioning/datasources/loki.yaml b/docker/telemetry/grafana/provisioning/datasources/loki.yaml new file mode 100644 index 00000000000..0ae48fe7094 --- /dev/null +++ b/docker/telemetry/grafana/provisioning/datasources/loki.yaml @@ -0,0 +1,35 @@ +# Grafana Loki data source provisioning for rippled log-trace correlation. +# +# Loki ingests rippled logs via OTel Collector's filelog receiver. +# The derivedFields config links trace_id values in log lines back to +# Tempo traces, enabling one-click log-to-trace navigation in Grafana. + +apiVersion: 1 + +datasources: + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + uid: loki + jsonData: + derivedFields: + # Trace ID is an OTel-issued 32-hex value emitted by Logs::format() + # as `trace_id=...`. Grafana treats the captured value as a Tempo + # trace ID, opening the trace directly via Tempo's trace-by-id API. + - datasourceUid: tempo + matcherRegex: "trace_id=(\\w+)" + name: TraceID + url: "$${__value.raw}" + # 64-char uppercase hex tokens in log bodies are XRPL ledger + # hashes (or tx hashes). They are NOT OTel trace IDs and cannot + # be resolved via the trace-by-id endpoint. Build a Grafana + # Explore deep-link that runs TraceQL on the span attribute + # `xrpl.consensus.ledger_id` — set on `consensus.round` spans to + # the full prev_ledger hash. This finds the round span whose + # first 16 bytes were folded into the OTel trace_id by + # SpanGuard::hashSpan(). + - matcherRegex: "\\b([A-F0-9]{64})\\b" + name: ConsensusLedgerHash + urlDisplayLabel: "Search Tempo" + url: '/explore?schemaVersion=1&orgId=1&panes={"tempo":{"datasource":"tempo","queries":[{"refId":"A","queryType":"traceql","query":"{.xrpl.consensus.ledger_id=\"$${__value.raw}\"}"}],"range":{"from":"now-1h","to":"now"}}}' diff --git a/docker/telemetry/grafana/provisioning/datasources/prometheus.yaml b/docker/telemetry/grafana/provisioning/datasources/prometheus.yaml new file mode 100644 index 00000000000..af7492822a3 --- /dev/null +++ b/docker/telemetry/grafana/provisioning/datasources/prometheus.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/docker/telemetry/grafana/provisioning/datasources/tempo.yaml b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml new file mode 100644 index 00000000000..f89fa89eec5 --- /dev/null +++ b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml @@ -0,0 +1,228 @@ +# Grafana datasource provisioning for Grafana Tempo. +# Auto-configures Tempo as a trace data source on Grafana startup. +# Access Grafana at http://localhost:3000, then use Explore -> Tempo +# to browse xrpld traces using TraceQL. +# +# Search filters provide pre-configured dropdowns in the Explore UI. +# Each phase adds filters for the span attributes it introduces. +# Base filters — node identity, service, span name, status. +# RPC command, status, role filters. +# Transaction hash, local/peer origin, status. +# Consensus mode, round, ledger sequence, close time. + +apiVersion: 1 + +datasources: + - name: Tempo + type: tempo + access: proxy + url: http://tempo:3200 + uid: tempo + jsonData: + nodeGraph: + enabled: true + # Service map and traces-to-metrics require a Prometheus datasource + # (not included in this stack). These features are inactive until a + # Prometheus service is added to docker-compose.yml. + serviceMap: + datasourceUid: prometheus + # Trace-to-log correlation — enables one-click navigation + # from a Tempo trace to the corresponding Loki log lines. Filters + # by trace_id so only logs from the same trace are shown. + tracesToLogs: + datasourceUid: loki + filterByTraceID: true + filterBySpanID: false + tags: [] + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: "-1h" + spanEndTimeShift: "1h" + search: + filters: + # --- Node identification filters --- + # service.name: logical service name (default: "xrpld"). + # Useful when running multiple service types in the same collector. + - id: service-name + tag: service.name + operator: "=" + scope: resource + type: static + # service.instance.id: unique node identifier — configurable via + # the service_instance_id setting in [telemetry], defaults to the + # node's public key. E.g. "Node-1" or "nHB1X37...". + - id: node-id + tag: service.instance.id + operator: "=" + scope: resource + type: dynamic + # service.version: xrpld build version (e.g., "2.4.0-b1"). + # Filter traces from specific software releases. + - id: node-version + tag: service.version + operator: "=" + scope: resource + type: dynamic + # xrpl.network.id: numeric network identifier + # (0 = mainnet, 1 = testnet, 2 = devnet, etc.). + # Derived from the [network_id] config section. + - id: network-id + tag: xrpl.network.id + operator: "=" + scope: resource + type: dynamic + # xrpl.network.type: human-readable network name derived from + # network ID ("mainnet", "testnet", "devnet", "unknown"). + - id: network-type + tag: xrpl.network.type + operator: "=" + scope: resource + type: dynamic + # --- Span intrinsic filters --- + # name: the span operation name (e.g., "rpc.command.server_info"). + # Use to find traces for a specific RPC command or subsystem. + - id: span-name + tag: name + operator: "=" + scope: intrinsic + type: dynamic + # status: span completion status ("ok", "error", "unset"). + # Filter for failed operations to diagnose errors. + - id: span-status + tag: status + operator: "=" + scope: intrinsic + type: dynamic + # duration: span wall-clock duration. Use with ">" operator + # to find slow operations (e.g., duration > 500ms). + - id: span-duration + tag: duration + operator: ">" + scope: intrinsic + type: dynamic + # RPC tracing filters + - id: rpc-command + tag: command + operator: "=" + scope: span + type: dynamic + - id: rpc-status + tag: rpc_status + operator: "=" + scope: span + type: dynamic + - id: rpc-role + tag: rpc_role + operator: "=" + scope: span + type: dynamic + # Transaction tracing filters + - id: tx-hash + tag: tx_hash + operator: "=" + scope: span + type: static + - id: tx-origin + tag: local + operator: "=" + scope: span + type: dynamic + - id: tx-status + tag: tx_status + operator: "=" + scope: span + type: dynamic + # tx_type: transaction type (e.g., "Payment", "OfferCreate"). + - id: tx-type + tag: tx_type + operator: "=" + scope: span + type: dynamic + # Consensus tracing filters + - id: consensus-mode + tag: consensus_mode + operator: "=" + scope: span + type: static + - id: consensus-round + tag: consensus_round + operator: "=" + scope: span + type: dynamic + - id: consensus-ledger-seq + tag: ledger_seq + operator: "=" + scope: span + type: static + # ledger_hash: ledger hash — scope all spans to a specific closed ledger. + - id: ledger-hash + tag: ledger_hash + operator: "=" + scope: span + type: static + - id: consensus-close-time-correct + tag: close_time_correct + operator: "=" + scope: span + type: static + - id: consensus-state + tag: consensus_state + operator: "=" + scope: span + type: static + - id: consensus-close-resolution + tag: close_resolution_ms + operator: "=" + scope: span + type: dynamic + - id: consensus-proposers + tag: proposers + operator: "=" + scope: span + type: dynamic + - id: consensus-result + tag: consensus_result + operator: "=" + scope: span + type: dynamic + - id: consensus-mode-old + tag: mode_old + operator: "=" + scope: span + type: dynamic + - id: consensus-mode-new + tag: mode_new + operator: "=" + scope: span + type: dynamic + - id: consensus-ledger-id + tag: consensus_ledger_id + operator: "=" + scope: span + type: static + # Additional transaction and queue filters + - id: tx-path + tag: path + operator: "=" + scope: span + type: dynamic + - id: tx-suppressed + tag: suppressed + operator: "=" + scope: span + type: dynamic + - id: peer-version + tag: peer_version + operator: "=" + scope: span + type: dynamic + - id: txq-status + tag: txq_status + operator: "=" + scope: span + type: dynamic + - id: txq-ter-code + tag: ter_code + operator: "=" + scope: span + type: dynamic diff --git a/docker/telemetry/integration-test.sh b/docker/telemetry/integration-test.sh new file mode 100755 index 00000000000..b4401d59060 --- /dev/null +++ b/docker/telemetry/integration-test.sh @@ -0,0 +1,756 @@ +#!/usr/bin/env bash +# Integration test for rippled OpenTelemetry instrumentation. +# +# Launches a 6-node xrpld consensus network with telemetry enabled, +# exercises RPC / transaction / consensus code paths, then verifies +# that the expected spans and metrics appear in Tempo and Prometheus. +# +# Usage: +# bash docker/telemetry/integration-test.sh +# +# Prerequisites: +# - .build/xrpld built with telemetry=ON +# - docker compose (v2) +# - curl, jq +# +# The script leaves the observability stack and xrpld nodes running +# so you can manually inspect Tempo (localhost:3200) and Grafana +# (localhost:3000). Run with --cleanup to tear down instead. + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +XRPLD="$REPO_ROOT/.build/xrpld" +COMPOSE_FILE="$SCRIPT_DIR/docker-compose.yml" +STANDALONE_CFG="$SCRIPT_DIR/xrpld-telemetry.cfg" +WORKDIR="${WORKDIR:-/tmp/xrpld-integration}" +NUM_NODES=6 +PEER_PORT_BASE=51235 +RPC_PORT_BASE=5005 +CONSENSUS_TIMEOUT=120 +GENESIS_ACCOUNT="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh" +GENESIS_SEED="snoPBrXtMeMyMHUVTgbuqAfg1SUTb" +DEST_ACCOUNT="" # Generated dynamically via wallet_propose +TEMPO="http://localhost:3200" +PROM="http://localhost:9090" + +# Counters for pass/fail +PASS=0 +FAIL=0 + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +log() { printf "\033[1;34m[INFO]\033[0m %s\n" "$*"; } +ok() { + printf "\033[1;32m[PASS]\033[0m %s\n" "$*" + PASS=$((PASS + 1)) +} +fail() { + printf "\033[1;31m[FAIL]\033[0m %s\n" "$*" + FAIL=$((FAIL + 1)) +} +die() { + printf "\033[1;31m[ERROR]\033[0m %s\n" "$*" >&2 + exit 1 +} + +check_span() { + local op="$1" + local count + count=$(curl -sf "$TEMPO/api/search" \ + --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \ + --data-urlencode "limit=5" | + jq '.traces | length' 2>/dev/null || echo 0) + if [ "$count" -gt 0 ]; then + ok "$op ($count traces)" + else + fail "$op (0 traces)" + fi +} + +# Verify trace_id injection in xrpld log output. +# Greps all node debug.log files for the "trace_id= span_id=" +# pattern that Logs::format() injects when an active OTel span exists. +# Also cross-checks that a trace_id found in logs matches a trace in Tempo. +check_log_correlation() { + log "Checking log-trace correlation..." + + local total_matches=0 + local files_scanned=0 + local sample_trace_id="" + + for i in $(seq 1 "$NUM_NODES"); do + local logfile="$WORKDIR/node$i/debug.log" + if [ ! -f "$logfile" ]; then + continue + fi + files_scanned=$((files_scanned + 1)) + local matches + matches=$(grep -c 'trace_id=[a-f0-9]\{32\} span_id=[a-f0-9]\{16\}' "$logfile") || matches=0 + total_matches=$((total_matches + matches)) + # Capture the first trace_id we find for cross-referencing with Tempo + if [ -z "$sample_trace_id" ] && [ "$matches" -gt 0 ]; then + sample_trace_id=$(grep -o 'trace_id=[a-f0-9]\{32\}' "$logfile" | head -1 | cut -d= -f2) + fi + done + + if [ "$files_scanned" -eq 0 ]; then + fail "Log correlation: no debug.log files found in $WORKDIR/node*/" + return + fi + + if [ "$total_matches" -gt 0 ]; then + ok "Log correlation: found $total_matches log lines with trace_id ($files_scanned nodes scanned)" + else + fail "Log correlation: no trace_id found in any node debug.log ($files_scanned nodes scanned)" + fi + + # Cross-check: verify the sample trace_id exists in Tempo + if [ -n "$sample_trace_id" ]; then + local trace_found + # Tempo /api/traces/{id} returns OTLP shape: {"batches":[...]} + trace_found=$(curl -sf "$TEMPO/api/traces/$sample_trace_id" | + jq '.batches | length' 2>/dev/null) || trace_found=0 + if [ "$trace_found" -gt 0 ]; then + ok "Log-Tempo cross-check: trace_id=$sample_trace_id found in Tempo" + else + fail "Log-Tempo cross-check: trace_id=$sample_trace_id NOT found in Tempo" + fi + fi +} + +cleanup() { + log "Cleaning up..." + # Kill xrpld nodes + for i in $(seq 1 "$NUM_NODES"); do + local pidfile="$WORKDIR/node$i/xrpld.pid" + if [ -f "$pidfile" ]; then + kill "$(cat "$pidfile")" 2>/dev/null || true + rm -f "$pidfile" + fi + done + # Also kill any straggling xrpld processes from our workdir + pkill -f "$WORKDIR" 2>/dev/null || true + # Stop docker stack + docker compose -f "$COMPOSE_FILE" down 2>/dev/null || true + # Remove workdir + rm -rf "$WORKDIR" + log "Cleanup complete." +} + +# Handle --cleanup flag +if [ "${1:-}" = "--cleanup" ]; then + cleanup + exit 0 +fi + +# --------------------------------------------------------------------------- +# Step 0: Prerequisites +# --------------------------------------------------------------------------- +log "Checking prerequisites..." + +command -v docker >/dev/null 2>&1 || die "docker not found" +docker compose version >/dev/null 2>&1 || die "docker compose (v2) not found" +command -v curl >/dev/null 2>&1 || die "curl not found" +command -v jq >/dev/null 2>&1 || die "jq not found" +[ -x "$XRPLD" ] || die "xrpld binary not found at $XRPLD (build with telemetry=ON)" +[ -f "$COMPOSE_FILE" ] || die "docker-compose.yml not found at $COMPOSE_FILE" +[ -f "$STANDALONE_CFG" ] || die "xrpld-telemetry.cfg not found at $STANDALONE_CFG" + +log "All prerequisites met." + +# --------------------------------------------------------------------------- +# Step 1: Clean previous run +# --------------------------------------------------------------------------- +log "Cleaning previous run data..." +for i in $(seq 1 "$NUM_NODES"); do + pidfile="$WORKDIR/node$i/xrpld.pid" + if [ -f "$pidfile" ]; then + kill "$(cat "$pidfile")" 2>/dev/null || true + fi +done +pkill -f "$WORKDIR" 2>/dev/null || true +# Kill any xrpld using the standalone config (from key generation) +pkill -f "xrpld-telemetry.cfg" 2>/dev/null || true +sleep 2 +rm -rf "$WORKDIR" +mkdir -p "$WORKDIR" + +# --------------------------------------------------------------------------- +# Step 2: Start observability stack +# --------------------------------------------------------------------------- +log "Starting observability stack..." +# Point the collector's log mount at this test's workdir so it tails the +# per-node debug.log files this script generates. The compose default +# (./data/logs) is for user-run xrpld; the test owns its own log root. +XRPLD_LOG_DIR="$WORKDIR" docker compose -f "$COMPOSE_FILE" up -d + +log "Waiting for otel-collector to be ready..." +for attempt in $(seq 1 30); do + # The OTLP HTTP endpoint returns 405 for GET (expects POST), which + # means it is listening. curl -sf would fail on 405, so we check + # the HTTP status code explicitly. + status=$(curl -so /dev/null -w '%{http_code}' http://localhost:4318/ 2>/dev/null || echo 000) + if [ "$status" != "000" ]; then + log "otel-collector ready (attempt $attempt, HTTP $status)." + break + fi + if [ "$attempt" -eq 30 ]; then + die "otel-collector not ready after 30s" + fi + sleep 1 +done + +log "Waiting for Tempo to be ready..." +for attempt in $(seq 1 30); do + if curl -sf "$TEMPO/ready" >/dev/null 2>&1; then + log "Tempo ready (attempt $attempt)." + break + fi + if [ "$attempt" -eq 30 ]; then + die "Tempo not ready after 30s" + fi + sleep 1 +done + +# --------------------------------------------------------------------------- +# Step 3: Generate validator keys +# --------------------------------------------------------------------------- +log "Generating $NUM_NODES validator key pairs..." + +# Start a temporary standalone xrpld for key generation +TEMP_DATA="$WORKDIR/temp-keygen" +mkdir -p "$TEMP_DATA" + +# Create a minimal temp config for key generation +TEMP_CFG="$TEMP_DATA/xrpld.cfg" +cat >"$TEMP_CFG" <"$TEMP_DATA/stdout.log" 2>&1 & +TEMP_PID=$! +log "Temporary xrpld started (PID $TEMP_PID), waiting for RPC..." + +for attempt in $(seq 1 30); do + if curl -sf http://localhost:5099 -d '{"method":"server_info"}' >/dev/null 2>&1; then + log "Temporary xrpld RPC ready (attempt $attempt)." + break + fi + if [ "$attempt" -eq 30 ]; then + kill "$TEMP_PID" 2>/dev/null || true + die "Temporary xrpld RPC not ready after 30s" + fi + sleep 1 +done + +declare -a SEEDS +declare -a PUBKEYS + +for i in $(seq 1 "$NUM_NODES"); do + result=$(curl -sf http://localhost:5099 -d '{"method":"validation_create"}') + seed=$(echo "$result" | jq -r '.result.validation_seed') + pubkey=$(echo "$result" | jq -r '.result.validation_public_key') + if [ -z "$seed" ] || [ "$seed" = "null" ]; then + kill "$TEMP_PID" 2>/dev/null || true + die "Failed to generate key pair $i" + fi + SEEDS+=("$seed") + PUBKEYS+=("$pubkey") + log " Node $i: $pubkey" +done + +kill "$TEMP_PID" 2>/dev/null || true +wait "$TEMP_PID" 2>/dev/null || true +rm -rf "$TEMP_DATA" +log "Key generation complete." + +# --------------------------------------------------------------------------- +# Step 4: Generate node configs and validators.txt +# --------------------------------------------------------------------------- +log "Generating node configs..." + +# Create shared validators.txt +VALIDATORS_FILE="$WORKDIR/validators.txt" +{ + echo "[validators]" + for i in $(seq 0 $((NUM_NODES - 1))); do + echo "${PUBKEYS[$i]}" + done +} >"$VALIDATORS_FILE" + +# Create per-node configs +for i in $(seq 1 "$NUM_NODES"); do + NODE_DIR="$WORKDIR/node$i" + mkdir -p "$NODE_DIR/nudb" "$NODE_DIR/db" + + RPC_PORT=$((RPC_PORT_BASE + i - 1)) + PEER_PORT=$((PEER_PORT_BASE + i - 1)) + SEED="${SEEDS[$((i - 1))]}" + + # Build ips_fixed list (all peers except self) + IPS_FIXED="" + for j in $(seq 1 "$NUM_NODES"); do + if [ "$j" -ne "$i" ]; then + IPS_FIXED="${IPS_FIXED}127.0.0.1 $((PEER_PORT_BASE + j - 1)) +" + fi + done + + cat >"$NODE_DIR/xrpld.cfg" <"$NODE_DIR/stdout.log" 2>&1 & + echo $! >"$NODE_DIR/xrpld.pid" + log " Node $i started (PID $(cat "$NODE_DIR/xrpld.pid"))" +done + +# Give nodes a moment to initialize +sleep 5 + +# --------------------------------------------------------------------------- +# Step 6: Wait for consensus +# --------------------------------------------------------------------------- +log "Waiting for nodes to reach 'proposing' state (timeout: ${CONSENSUS_TIMEOUT}s)..." + +start_time=$(date +%s) +nodes_ready=0 + +while [ "$nodes_ready" -lt "$NUM_NODES" ]; do + elapsed=$(($(date +%s) - start_time)) + if [ "$elapsed" -ge "$CONSENSUS_TIMEOUT" ]; then + fail "Consensus timeout after ${CONSENSUS_TIMEOUT}s ($nodes_ready/$NUM_NODES nodes ready)" + log "Continuing with partial consensus..." + break + fi + + nodes_ready=0 + for i in $(seq 1 "$NUM_NODES"); do + RPC_PORT=$((RPC_PORT_BASE + i - 1)) + state=$(curl -sf "http://localhost:$RPC_PORT" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.server_state' 2>/dev/null || echo "unreachable") + if [ "$state" = "proposing" ]; then + nodes_ready=$((nodes_ready + 1)) + fi + done + printf "\r %d/%d nodes proposing (%ds elapsed)..." "$nodes_ready" "$NUM_NODES" "$elapsed" + if [ "$nodes_ready" -lt "$NUM_NODES" ]; then + sleep 3 + fi +done +echo "" + +if [ "$nodes_ready" -eq "$NUM_NODES" ]; then + ok "All $NUM_NODES nodes reached 'proposing' state" +else + fail "Only $nodes_ready/$NUM_NODES nodes reached 'proposing' state" +fi + +# --------------------------------------------------------------------------- +# Step 6b: Wait for validated ledger +# --------------------------------------------------------------------------- +log "Waiting for first validated ledger..." +for attempt in $(seq 1 60); do + val_seq=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) + if [ "$val_seq" -gt 2 ] 2>/dev/null; then + ok "First validated ledger: seq $val_seq" + break + fi + if [ "$attempt" -eq 60 ]; then + fail "No validated ledger after 60s" + fi + sleep 1 +done + +# --------------------------------------------------------------------------- +# Step 7: Exercise RPC spans (Phase 2) +# --------------------------------------------------------------------------- +log "Exercising RPC spans..." + +curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d '{"method":"server_info"}' >/dev/null +curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d '{"method":"server_state"}' >/dev/null +curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' >/dev/null + +log "RPC commands sent. Waiting 5s for batch export..." +sleep 5 + +# --------------------------------------------------------------------------- +# Step 8: Submit transaction (Phase 3) +# --------------------------------------------------------------------------- +log "Submitting Payment transaction..." + +# Generate a destination wallet +log " Generating destination wallet..." +wallet_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d '{"method":"wallet_propose"}') +DEST_ACCOUNT=$(echo "$wallet_result" | jq -r '.result.account_id' 2>/dev/null) +if [ -z "$DEST_ACCOUNT" ] || [ "$DEST_ACCOUNT" = "null" ]; then + fail "Could not generate destination wallet" + DEST_ACCOUNT="rrrrrrrrrrrrrrrrrrrrrhoLvTp" # ACCOUNT_ZERO fallback +fi +log " Destination: $DEST_ACCOUNT" + +# Get genesis account info +acct_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d "{\"method\":\"account_info\",\"params\":[{\"account\":\"$GENESIS_ACCOUNT\"}]}") +seq_num=$(echo "$acct_result" | jq -r '.result.account_data.Sequence' 2>/dev/null || echo "unknown") +log " Genesis account sequence: $seq_num" + +# Submit payment +submit_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d "{\"method\":\"submit\",\"params\":[{\"secret\":\"$GENESIS_SEED\",\"tx_json\":{\"TransactionType\":\"Payment\",\"Account\":\"$GENESIS_ACCOUNT\",\"Destination\":\"$DEST_ACCOUNT\",\"Amount\":\"10000000\"}}]}") + +engine_result=$(echo "$submit_result" | jq -r '.result.engine_result' 2>/dev/null || echo "unknown") +tx_hash=$(echo "$submit_result" | jq -r '.result.tx_json.hash' 2>/dev/null || echo "unknown") + +if [ "$engine_result" = "tesSUCCESS" ] || [ "$engine_result" = "terQUEUED" ]; then + ok "Transaction submitted: $engine_result (hash: ${tx_hash:0:16}...)" +else + fail "Transaction submission: $engine_result" + log " Full response: $(echo "$submit_result" | jq -c .result 2>/dev/null)" +fi + +log "Waiting 15s for consensus round + batch export..." +sleep 15 + +# --------------------------------------------------------------------------- +# Step 9: Verify Tempo traces +# --------------------------------------------------------------------------- +log "Verifying spans in Tempo..." + +# Check service registration +services=$(curl -sf "$TEMPO/api/v2/search/tag/resource.service.name/values" | + jq -r '.tagValues[].value' 2>/dev/null || echo "") +if echo "$services" | grep -q "xrpld"; then + ok "Service 'xrpld' registered in Tempo" +else + fail "Service 'xrpld' NOT found in Tempo (found: $services)" +fi + +log "" +log "--- Phase 2: RPC Spans ---" +check_span "rpc.request" +check_span "rpc.process" +check_span "rpc.command.server_info" +check_span "rpc.command.server_state" +check_span "rpc.command.ledger" + +log "" +log "--- Phase 3: Transaction Spans ---" +check_span "tx.process" +check_span "tx.receive" +check_span "tx.apply" + +log "" +log "--- Phase 4: Consensus Spans ---" +check_span "consensus.proposal.send" +check_span "consensus.ledger_close" +check_span "consensus.accept" +check_span "consensus.validation.send" + +log "" +log "--- Phase 5: Ledger Spans ---" +check_span "ledger.build" +check_span "ledger.validate" +check_span "ledger.store" + +log "" +log "--- Phase 5: Peer Spans (trace_peer=1) ---" +check_span "peer.proposal.receive" +check_span "peer.validation.receive" + +# --------------------------------------------------------------------------- +# Step 9b: Verify log-trace correlation (Phase 8) +# --------------------------------------------------------------------------- +log "" +log "--- Log-Trace Correlation ---" +check_log_correlation + +# --------------------------------------------------------------------------- +# Step 10: Verify Prometheus spanmetrics +# --------------------------------------------------------------------------- +log "" +log "--- Phase 5: Spanmetrics ---" +log "Waiting 20s for Prometheus scrape cycle..." +sleep 20 + +calls_count=$(curl -sf "$PROM/api/v1/query?query=span_calls_total" | + jq '.data.result | length' 2>/dev/null || echo 0) +if [ "$calls_count" -gt 0 ]; then + ok "Prometheus: span_calls_total ($calls_count series)" +else + fail "Prometheus: span_calls_total (0 series)" +fi + +duration_count=$(curl -sf "$PROM/api/v1/query?query=span_duration_milliseconds_count" | + jq '.data.result | length' 2>/dev/null || echo 0) +if [ "$duration_count" -gt 0 ]; then + ok "Prometheus: duration histogram ($duration_count series)" +else + fail "Prometheus: duration histogram (0 series)" +fi + +# Check Grafana +if curl -sf http://localhost:3000/api/health >/dev/null 2>&1; then + ok "Grafana: healthy at localhost:3000" +else + fail "Grafana: not reachable at localhost:3000" +fi + +# --------------------------------------------------------------------------- +# Step 10b: Verify native OTel metrics in Prometheus (beast::insight) +# --------------------------------------------------------------------------- +log "" +log "--- Phase 7: Native OTel Metrics (beast::insight via OTLP) ---" +log "Waiting 20s for OTLP metric export + Prometheus scrape..." +sleep 20 + +check_otel_metric() { + local metric_name="$1" + local result + result=$(curl -sf "$PROM/api/v1/query?query=$metric_name" | + jq '.data.result | length' 2>/dev/null || echo 0) + if [ "$result" -gt 0 ]; then + ok "OTel: $metric_name ($result series)" + else + fail "OTel: $metric_name (0 series)" + fi +} + +# Node health gauges (ObservableGauge — no _total suffix) +check_otel_metric "ledgermaster_validated_ledger_age" +check_otel_metric "ledgermaster_published_ledger_age" +check_otel_metric "job_count" + +# State accounting +check_otel_metric "state_accounting_full_duration" + +# Peer finder +check_otel_metric "peer_finder_active_inbound_peers" +check_otel_metric "peer_finder_active_outbound_peers" + +# RPC counters (Counter — Prometheus adds _total suffix automatically) +check_otel_metric "rpc_requests_total" + +# Overlay traffic +check_otel_metric "total_bytes_in" + +# Verify StatsD receiver is NOT required (no statsd receiver in pipeline) +log "" +log "--- Verify StatsD receiver is not required ---" +# StatsD listens on UDP 8125, so probe with a UDP-aware tool, not curl (TCP). +if command -v ss >/dev/null 2>&1; then + if ss -ulnp 2>/dev/null | grep -q ":8125"; then + fail "StatsD port 8125 appears to be listening (should not be needed)" + else + ok "StatsD port 8125 is not listening (not required)" + fi +else + log "ss not found -- skipping StatsD UDP port check" +fi + +# --------------------------------------------------------------------------- +# Step 10c: Verify Phase 9 OTel SDK Metrics +# --------------------------------------------------------------------------- +log "" +log "--- Phase 9: OTel SDK Metrics (MetricsRegistry) ---" +log "Waiting 15s for OTel metric export + Prometheus scrape..." +sleep 15 + +check_otel_metric() { + local metric_name="$1" + local result + result=$(curl -sf "$PROM/api/v1/query?query=$metric_name" | + jq '.data.result | length' 2>/dev/null || echo 0) + if [ "$result" -gt 0 ]; then + ok "OTel: $metric_name ($result series)" + else + fail "OTel: $metric_name (0 series)" + fi +} + +# Task 9.1: NodeStore I/O +check_otel_metric 'nodestore_state{metric="node_reads_total"}' +check_otel_metric 'nodestore_state{metric="write_load"}' + +# Task 9.2: Cache hit rates +check_otel_metric 'cache_metrics{metric="SLE_hit_rate"}' +check_otel_metric 'cache_metrics{metric="treenode_cache_size"}' + +# Task 9.3: TxQ metrics +check_otel_metric 'txq_metrics{metric="txq_count"}' +check_otel_metric 'txq_metrics{metric="txq_reference_fee_level"}' + +# Task 9.4: Per-RPC metrics +check_otel_metric "rpc_method_started_total" +check_otel_metric "rpc_method_finished_total" + +# Task 9.5: Per-job metrics +check_otel_metric "job_queued_total" +check_otel_metric "job_finished_total" + +# Task 9.6: Counted object instances +check_otel_metric "object_count" + +# Task 9.7: Load factor breakdown +check_otel_metric 'load_factor_metrics{metric="load_factor"}' +check_otel_metric 'load_factor_metrics{metric="load_factor_server"}' + +# Task 7.15 / Phase 9: ValidationTracker rolling-window agreement gauge. +# MetricsRegistry::registerValidationAgreementGauge() publishes +# validation_agreement with a `metric` label for each window +# (1h / 24h / 7d) plus the matching agreement/miss counts. The 7-day +# window matches the external xrpl-validator-dashboard parity target. +check_otel_metric 'validation_agreement{metric="agreement_pct_1h"}' +check_otel_metric 'validation_agreement{metric="agreement_pct_24h"}' +check_otel_metric 'validation_agreement{metric="agreement_pct_7d"}' +check_otel_metric 'validation_agreement{metric="agreements_1h"}' +check_otel_metric 'validation_agreement{metric="missed_1h"}' +check_otel_metric 'validation_agreement{metric="agreements_24h"}' +check_otel_metric 'validation_agreement{metric="missed_24h"}' +check_otel_metric 'validation_agreement{metric="agreements_7d"}' +check_otel_metric 'validation_agreement{metric="missed_7d"}' + +# --------------------------------------------------------------------------- +# Step 11: Summary +# --------------------------------------------------------------------------- +echo "" +echo "===========================================================" +echo " INTEGRATION TEST RESULTS" +echo "===========================================================" +printf " \033[1;32mPASSED: %d\033[0m\n" "$PASS" +printf " \033[1;31mFAILED: %d\033[0m\n" "$FAIL" +echo "===========================================================" +echo "" +echo " Observability stack is running:" +echo "" +echo " Tempo: http://localhost:3200" +echo " Grafana: http://localhost:3000" +echo " Prometheus: http://localhost:9090" +echo " Loki: http://localhost:3100" +echo "" +echo " xrpld nodes (6) are running:" +for i in $(seq 1 "$NUM_NODES"); do + RPC_PORT=$((RPC_PORT_BASE + i - 1)) + PEER_PORT=$((PEER_PORT_BASE + i - 1)) + echo " Node $i: RPC=localhost:$RPC_PORT Peer=:$PEER_PORT PID=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null || echo 'unknown')" +done +echo "" +echo " To tear down:" +echo " bash docker/telemetry/integration-test.sh --cleanup" +echo "" +echo "===========================================================" + +if [ "$FAIL" -gt 0 ]; then + exit 1 +fi diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml new file mode 100644 index 00000000000..20b93c7c615 --- /dev/null +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -0,0 +1,271 @@ +# OpenTelemetry Collector configuration — Grafana Cloud dual-export variant. +# +# Identical to otel-collector-config.yaml (local Tempo/Prometheus/Loki) but +# ALSO ships all three signals to Grafana Cloud over a single OTLP/HTTP +# endpoint. Local backends are kept so the on-box stack still works as a +# fallback; remove the local exporters from the pipelines below if you want +# cloud-only. +# +# Selecting this config is a RUNTIME choice — no code or image changes: +# Local only (default): +# docker compose -f docker/telemetry/docker-compose.yml up -d +# Local + Grafana Cloud: +# docker compose -f docker/telemetry/docker-compose.yml \ +# -f docker/telemetry/docker-compose.grafanacloud.yaml up -d +# +# The override file mounts THIS config over the collector's config path and +# injects the three required secrets as environment variables: +# GRAFANA_CLOUD_OTLP_ENDPOINT e.g. https://otlp-gateway-.grafana.net/otlp +# GRAFANA_CLOUD_INSTANCE_ID numeric Grafana Cloud stack/instance id +# GRAFANA_CLOUD_API_TOKEN a Cloud Access Policy token with metrics/traces/logs:write +# Find all three under Grafana Cloud -> Connections -> "OpenTelemetry (OTLP)". + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + # Basic-auth for the Grafana Cloud OTLP gateway: username = instance id, + # password = API token. Values come from the container environment so no + # secret is committed to the repo. + basicauth/grafanacloud: + client_auth: + username: ${env:GRAFANA_CLOUD_INSTANCE_ID} + password: ${env:GRAFANA_CLOUD_API_TOKEN} + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + filelog: + include: + - /var/log/xrpld/*/debug.log + operators: + - type: regex_parser + regex: '^(?P\S+\s+\S+)\s+\S+\s+(?:(?P\S+):)?(?P\S+)\s+(?:trace_id=(?P[a-f0-9]+)\s+span_id=(?P[a-f0-9]+)\s+)?(?P.*)$' + timestamp: + parse_from: attributes.timestamp + layout: "%Y-%b-%d %H:%M:%S.%f" + location: UTC + +processors: + batch: + timeout: 1s + send_batch_size: 100 + # Box-only (uncommitted): probabilistic 0.5% tail sampling on the + # trace-storage branch only. spanmetrics-derived metrics see 100% of spans + # (they run on a separate branch) so RED remain exact; Tempo/Cloud stores + # ~1/200 of traces to keep long-running node cost bounded. + tail_sampling: + decision_wait: 10s + num_traces: 50000 + policies: + - name: half-percent + type: probabilistic + probabilistic: + sampling_percentage: 0.5 + resource/logs: + attributes: + - key: service.name + value: xrpld + action: upsert + - key: job + value: xrpld + action: upsert + # Deployment-tier tagging. Each collector serves ONE environment and ONE + # network, so it stamps both onto every signal it forwards to Grafana + # Cloud, letting one cloud stack hold data from many collectors and + # filter by tier. + # - deployment.environment: the collector IS the environment (local, ci, + # test, prod), so it is authoritative -> upsert (overwrite). + # - xrpl.network.type: the xrpld node knows its own chain and already + # stamps this, so the collector only fills it when absent -> insert. + # This keeps a node's real network (e.g. a local node on mainnet) + # from being overwritten by a collector's default. + # Replace the placeholder values per collector; see the "Deployment + # Tiers" section of docs/telemetry-runbook.md. + resource/tier: + attributes: + - key: deployment.environment + value: local + action: upsert + - key: xrpl.network.type + value: mainnet + action: insert + # Strip SDK-injected resource attributes (telemetry.sdk.language/name/version). + # The OpenTelemetry SDK auto-adds these to every Resource; they carry no + # operational value, so drop them for every signal on every backend. + resource/stripsdk: + attributes: + - key: telemetry.sdk.language + action: delete + - key: telemetry.sdk.name + action: delete + - key: telemetry.sdk.version + action: delete + # Grafana Cloud ingests metrics via OTLP (no Prometheus scrape), so the + # tier/instance resource attributes never become series labels the way the + # local prometheus exporter promotes them (resource_to_telemetry_conversion + # is Prometheus-exporter-only). Copy them onto datapoint labels so the + # dashboards' $node / $deployment_environment / $xrpl_network_type filters + # resolve on Cloud exactly as they do locally. + # + # NOTE: service.name is intentionally NOT copied here. Cloud's OTLP ingest + # already promotes service.name to the `service_name` label, and adding a + # datapoint attribute with the same name causes Mimir to concatenate both + # values as `xrpld;xrpld`. service.instance.id does not exhibit the same + # doubling, so the copy for $node is kept. + transform/cloudlabels: + metric_statements: + - context: datapoint + statements: + - set(attributes["service_instance_id"], resource.attributes["service.instance.id"]) + - set(attributes["deployment_environment"], resource.attributes["deployment.environment"]) + - set(attributes["xrpl_network_type"], resource.attributes["xrpl.network.type"]) + +connectors: + spanmetrics: + namespace: "span" + # Emit span-derived metrics every 15s (default is 60s). Grafana Cloud + # ingests these as one datapoint per flush, and dashboard rate() panels + # use [$__rate_interval], which at short ranges (<=1h) can be shorter than + # the flush interval. A 60s flush left <2 samples per rate window on Cloud, + # so those panels rendered "No data". 15s keeps >=2 samples per window at + # every dashboard range, at the cost of ~4x span-metric datapoints to Cloud. + metrics_flush_interval: 15s + # Resource attributes that define the resource-metrics grouping key. + # All resource attributes propagate to the output metrics regardless; + # label promotion is handled by resource_to_telemetry_conversion on the + # prometheus exporter. Listing the per-node id and tier keys keeps + # series from distinct nodes/tiers grouped separately. + resource_metrics_key_attributes: + - service.instance.id + - deployment.environment + - xrpl.network.type + histogram: + # Pin unit=ms so the metric stays span_duration_milliseconds_* and le + # labels stay in ms even if a future collector flips the default to + # seconds (connector.spanmetrics.useSecondAsDefaultMetricsUnit gate). + unit: ms + explicit: + # Buckets MUST stay strictly ascending (the connector binary-searches + # them and silently misbuckets otherwise). Sub-second boundaries cover + # RPC/tx/ledger spans; the 2s-4s boundaries resolve second-scale + # consensus spans (consensus.round ~3.9s, consensus.establish ~1.9s) + # that used to pile into a single 1s-5s bucket; 10s/30s give the + # ledger.acquire tail (~17% exceeds 5s during catch-up) a measurable + # home so its p95/p99 stop reading as +Inf. + buckets: + [ + 1ms, + 5ms, + 10ms, + 25ms, + 50ms, + 100ms, + 250ms, + 500ms, + 1s, + 2s, + 3s, + 4s, + 5s, + 10s, + 30s, + ] + dimensions: + - name: command + - name: rpc_status + - name: consensus_mode + # Only the boolean close_time_correct is promoted, never the close_time + # value itself: close_time is a monotonic NetClock timestamp, so a + # dimension on it would mint a new metric series every ledger (unbounded + # cardinality). The close-time value is exposed as the server_info + # last_close_time gauge instead (for last-close age); the close interval + # comes from the ledgers_closed_total counter. + - name: close_time_correct + - name: consensus_state + - name: local + - name: suppressed + - name: proposal_trusted + - name: validation_trusted + - name: tx_type + - name: ter_result + - name: stage + - name: txq_status + - name: load_type + - name: is_batch + - name: mode_new + - name: consensus_stalled + - name: consensus_phase + - name: consensus_result + - name: method + - name: grpc_role + - name: grpc_status + - name: outcome + - name: acquire_reason + +exporters: + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + otlphttp/loki: + endpoint: http://loki:3100/otlp + prometheus: + endpoint: 0.0.0.0:8889 + # Promote resource attributes (deployment.environment, xrpl.network.type, + # service.name, service.instance.id) to Prometheus labels so the local + # scrape surface can be filtered by tier. (Grafana Cloud ingests the + # attributes natively via OTLP, so this only affects the local export.) + resource_to_telemetry_conversion: + enabled: true + # Single OTLP/HTTP exporter to Grafana Cloud. The gateway fans the three + # signals out to hosted Tempo (traces), Mimir/Prometheus (metrics), and + # Loki (logs). Retry + queue guard against transient gateway errors. + otlphttp/grafanacloud: + endpoint: ${env:GRAFANA_CLOUD_OTLP_ENDPOINT} + auth: + authenticator: basicauth/grafanacloud + retry_on_failure: + enabled: true + sending_queue: + enabled: true + +service: + extensions: [health_check, basicauth/grafanacloud] + pipelines: + # Each pipeline keeps its local exporter(s) AND adds Grafana Cloud. + # For cloud-only, drop debug/otlp/tempo, prometheus, and otlphttp/loki + # from the respective exporter lists. + # 100% of spans feed the spanmetrics connector so span-derived RED + # metrics stay exact. No tail sampling on this branch. + traces/metrics: + receivers: [otlp] + processors: [resource/tier, resource/stripsdk, batch] + exporters: [spanmetrics] + # Trace-STORAGE branch: 0.5% probabilistic tail sampling before Tempo + # and Grafana Cloud, so stored trace volume is ~1/200 of ingested spans. + traces/store: + receivers: [otlp] + processors: [tail_sampling, resource/tier, resource/stripsdk, batch] + exporters: [otlp/tempo, otlphttp/grafanacloud] + # The local Prometheus scrape promotes tier/instance resource attrs to + # labels via resource_to_telemetry_conversion; Grafana Cloud (OTLP) does + # not, so it runs a separate pipeline that copies them onto datapoint + # labels via transform/cloudlabels. Splitting avoids the local scrape and + # the transform both writing a service_instance_id label on the same series. + metrics/local: + receivers: [otlp, spanmetrics] + processors: [resource/tier, resource/stripsdk, batch] + exporters: [prometheus] + metrics/cloud: + receivers: [otlp, spanmetrics] + processors: + [resource/tier, resource/stripsdk, transform/cloudlabels, batch] + exporters: [otlphttp/grafanacloud] + logs: + receivers: [filelog] + processors: [resource/logs, resource/tier, resource/stripsdk, batch] + exporters: [otlphttp/loki, otlphttp/grafanacloud] diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml new file mode 100644 index 00000000000..03ddeac31ec --- /dev/null +++ b/docker/telemetry/otel-collector-config.yaml @@ -0,0 +1,230 @@ +# OpenTelemetry Collector configuration for xrpld development. +# +# Pipelines: +# traces: OTLP receiver -> batch processor -> debug + Tempo + spanmetrics +# metrics: OTLP receiver + spanmetrics connector -> Prometheus exporter +# logs: filelog receiver -> batch processor -> otlphttp/Loki (Phase 8) +# +# xrpld sends traces via OTLP/HTTP to port 4318. The collector batches +# them, forwards to Tempo, and derives RED metrics via the spanmetrics +# connector, which Prometheus scrapes on port 8889. +# +# xrpld sends beast::insight metrics natively via OTLP/HTTP to port 4318 +# (same endpoint as traces). The OTLP receiver feeds both the traces and +# metrics pipelines. Metrics are exported to Prometheus alongside +# span-derived metrics. +# +# The filelog receiver tails xrpld's debug.log files under +# /var/log/xrpld/ (mounted from the host). A regex_parser operator +# extracts timestamp, partition, severity, and optional trace_id/span_id +# fields injected by Logs::format(). Parsed logs are exported to Grafana +# Loki for log-trace correlation. + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + # Filelog receiver tails xrpld debug.log files for log-trace + # correlation. Extracts structured fields (timestamp, partition, severity, + # trace_id, span_id, message) via regex. The trace_id and span_id are + # optional — only present when the log was emitted within an active span. + filelog: + include: [/var/log/xrpld/*/debug.log] + operators: + # Log format emitted by Logs::format() is: + # YYYY-Mmm-DD HH:MM:SS.ffffff UTC : [trace_id=... span_id=...] + # The `partition:` prefix is omitted when partition is empty, so the + # capture group is non-capturing optional. Fractional seconds up to 6 + # digits are parsed via the `%f` strptime directive. + - type: regex_parser + regex: '^(?P\S+\s+\S+)\s+\S+\s+(?:(?P\S+):)?(?P\S+)\s+(?:trace_id=(?P[a-f0-9]+)\s+span_id=(?P[a-f0-9]+)\s+)?(?P.*)$' + timestamp: + parse_from: attributes.timestamp + layout: "%Y-%b-%d %H:%M:%S.%f" + location: UTC + +processors: + batch: + timeout: 1s + send_batch_size: 100 + resource/logs: + attributes: + - key: service.name + value: xrpld + action: upsert + # Loki 3.x OTLP ingestion converts `service.name` to the label + # `service_name`. The runbook and integration-test queries use the + # canonical Loki label `job` so operators can paste `{job="xrpld"}` + # without guessing the otel-to-loki naming convention. Upsert the + # `job` resource attribute here so it round-trips through OTLP + # into Loki as the `job` label. + - key: job + value: xrpld + action: upsert + # Deployment-tier tagging. Each collector serves ONE environment and ONE + # network, so it stamps both onto every signal it forwards. This lets a + # single Grafana stack hold data from many collectors and filter by tier. + # - deployment.environment: the collector IS the environment (local, ci, + # test, prod), so it is authoritative -> upsert (overwrite). + # - xrpl.network.type: the xrpld node knows its own chain and already + # stamps this, so the collector only fills it when absent -> insert. + # This keeps a node's real network (e.g. a local node on mainnet) + # from being overwritten by a collector's default. + # Replace the placeholder values per collector; see docker/telemetry + # tier examples. + resource/tier: + attributes: + - key: deployment.environment + value: local + action: upsert + - key: xrpl.network.type + value: mainnet + action: insert + # Strip SDK-injected resource attributes (telemetry.sdk.language/name/version). + # The OpenTelemetry SDK auto-adds these to every Resource; they carry no + # operational value and clutter the attribute set on every backend, so drop + # them here for all signals. + resource/stripsdk: + attributes: + - key: telemetry.sdk.language + action: delete + - key: telemetry.sdk.name + action: delete + - key: telemetry.sdk.version + action: delete + # Defense-in-depth: hash path-finding account attributes. The xrpld SDK + # already hashes these before export, but a node that emitted raw values + # is caught here so raw addresses never reach the backend. + attributes/hash: + actions: + - key: pathfind_source_account + action: hash + - key: pathfind_dest_account + action: hash + +connectors: + spanmetrics: + namespace: "span" + # Resource attributes that define the resource-metrics grouping key. + # All resource attributes are propagated onto the output metrics + # regardless; promotion to Prometheus labels is handled by the + # prometheus exporter's resource_to_telemetry_conversion. Listing the + # per-node id and the deployment-tier keys keeps series from distinct + # nodes/tiers grouped separately. + resource_metrics_key_attributes: + - service.instance.id + - deployment.environment + - xrpl.network.type + histogram: + # Pin unit=ms so the metric stays span_duration_milliseconds_* and le + # labels stay in ms even if a future collector flips the default to + # seconds (connector.spanmetrics.useSecondAsDefaultMetricsUnit gate). + unit: ms + explicit: + # Buckets MUST stay strictly ascending (the connector binary-searches + # them and silently misbuckets otherwise). Sub-second boundaries cover + # RPC/tx/ledger spans; the 2s-4s boundaries resolve second-scale + # consensus spans (consensus.round ~3.9s, consensus.establish ~1.9s) + # that used to pile into a single 1s-5s bucket; 10s/30s give the + # ledger.acquire tail (~17% exceeds 5s during catch-up) a measurable + # home so its p95/p99 stop reading as +Inf. + buckets: + [ + 1ms, + 5ms, + 10ms, + 25ms, + 50ms, + 100ms, + 250ms, + 500ms, + 1s, + 2s, + 3s, + 4s, + 5s, + 10s, + 30s, + ] + dimensions: + - name: command + - name: rpc_status + - name: consensus_mode + # Only the boolean close_time_correct is promoted, never the close_time + # value itself: close_time is a monotonic NetClock timestamp, so a + # dimension on it would mint a new metric series every ledger (unbounded + # cardinality). The close-time value is exposed as the server_info + # last_close_time gauge instead (for last-close age); the close interval + # comes from the ledgers_closed_total counter. + - name: close_time_correct + - name: consensus_state + - name: local + - name: suppressed + - name: proposal_trusted + - name: validation_trusted + - name: tx_type + - name: ter_result + # Apply-pipeline stage (preflight|preclaim|apply) — splits the + # tx.preflight/tx.preclaim/tx.transactor span RED metrics per stage. + - name: stage + - name: txq_status + - name: load_type + - name: is_batch + # Consensus lifecycle dimensions (low cardinality, bounded value sets). + - name: mode_new + - name: consensus_stalled + - name: consensus_phase + - name: consensus_result + # gRPC surface dimensions (bounded: method names, role, status). + - name: method + - name: grpc_role + - name: grpc_status + # ledger.acquire dimensions (bounded: outcome, acquire reason). + - name: outcome + - name: acquire_reason + +exporters: + debug: + verbosity: detailed + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + # Export logs to Grafana Loki via OTLP/HTTP. Loki 3.x supports + # native OTLP ingestion on its /otlp endpoint, replacing the removed + # loki exporter (dropped in otel-collector-contrib v0.147.0). + otlphttp/loki: + endpoint: http://loki:3100/otlp + prometheus: + endpoint: 0.0.0.0:8889 + # Promote resource attributes (deployment.environment, xrpl.network.type, + # service.name, service.instance.id) to Prometheus labels. Without this + # the tier attributes stamped by resource/tier stay on the resource and + # never reach the metric series, so dashboards could not filter by them. + resource_to_telemetry_conversion: + enabled: true + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [resource/tier, resource/stripsdk, attributes/hash, batch] + exporters: [debug, otlp/tempo, spanmetrics] + metrics: + receivers: [otlp, spanmetrics] + processors: [resource/tier, resource/stripsdk, batch] + exporters: [prometheus] + # Log pipeline ingests xrpld debug.log via filelog receiver, + # batches entries, and exports to Loki for log-trace correlation. + logs: + receivers: [filelog] + processors: [resource/logs, resource/tier, resource/stripsdk, batch] + exporters: [otlphttp/loki] diff --git a/docker/telemetry/prometheus.yml b/docker/telemetry/prometheus.yml new file mode 100644 index 00000000000..d99d919a556 --- /dev/null +++ b/docker/telemetry/prometheus.yml @@ -0,0 +1,9 @@ +# Prometheus configuration for scraping spanmetrics from OTel Collector. +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8889"] diff --git a/docker/telemetry/tempo.yaml b/docker/telemetry/tempo.yaml new file mode 100644 index 00000000000..24c092ca95f --- /dev/null +++ b/docker/telemetry/tempo.yaml @@ -0,0 +1,84 @@ +# Grafana Tempo configuration for xrpld telemetry stack. +# +# Runs in single-binary mode for local development. +# Receives traces via OTLP/gRPC from the OTel Collector and stores +# them locally. Queryable via Grafana Explore using the Tempo datasource. +# +# Search filters are configured on the Grafana datasource side +# (grafana/provisioning/datasources/tempo.yaml). Tempo auto-indexes +# all span attributes for search in single-binary mode. +# +# For production, replace local storage with S3/GCS backend and adjust +# retention via the compactor settings. See: +# https://grafana.com/docs/tempo/latest/configuration/ + +stream_over_http_enabled: true + +server: + http_listen_port: 3200 + +# Raise the TraceQL metrics query range limit. The default +# query_frontend.metrics.max_duration is 3h, so a dashboard set to a longer +# window (e.g. 6h/12h) fails with "range exceeds 3h0m0s". 168h matches the +# search max_duration and gives dashboards generous headroom. +query_frontend: + metrics: + max_duration: 168h + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 1h + +# Enable metrics generator for service graph, span metrics, and the +# local-blocks processor. Produces RED metrics (rate, errors, duration) per +# service/span for the service map, and keeps recent trace blocks queryable so +# TraceQL metrics queries (quantile_over_time, count_over_time, etc. via +# /api/metrics/query_range) work. +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + # Uncomment and add a Prometheus service to docker-compose.yml + # to enable remote_write for service graph metrics: + # remote_write: + # - url: http://prometheus:9090/api/v1/write + # Separate WAL the local-blocks processor flushes traces to for metrics + # queries. Required when flush_to_storage is true. + traces_storage: + path: /var/tempo/generator/traces + processor: + local_blocks: + # xrpld consensus/transaction spans are SPAN_KIND_INTERNAL. By default + # local-blocks keeps only server spans for TraceQL metrics, so attribute + # aggregations over internal spans return nothing. Keep all spans. + filter_server_spans: false + # Flush recent blocks to traces_storage so query_range can read them. + flush_to_storage: true + +overrides: + defaults: + metrics_generator: + processors: + - service-graphs + - span-metrics + - local-blocks + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks diff --git a/docker/telemetry/validators-devnet.txt b/docker/telemetry/validators-devnet.txt new file mode 100644 index 00000000000..644dac96f28 --- /dev/null +++ b/docker/telemetry/validators-devnet.txt @@ -0,0 +1,11 @@ +# Devnet validator list configuration. +# +# Uses the Devnet validator list publisher operated by Ripple. +# This fetches the current set of trusted validators for the +# XRP Ledger Devnet automatically. + +[validator_list_sites] +https://vl.devnet.rippletest.net + +[validator_list_keys] +EDBB54B0D9AEE071BB37784AF5A9E7CC49AC7A0EFCE868C54532BCB966B9CFC13B diff --git a/docker/telemetry/validators-mainnet.txt b/docker/telemetry/validators-mainnet.txt new file mode 100644 index 00000000000..ca908e237ac --- /dev/null +++ b/docker/telemetry/validators-mainnet.txt @@ -0,0 +1,15 @@ +# Mainnet validator list configuration. +# +# Uses the production XRPL validator list publishers operated by +# Ripple and the XRPL Foundation. These fetch the current set of +# trusted validators for XRPL Mainnet automatically. + +[validator_list_sites] +https://vl.ripple.com +https://unl.xrplf.org + +[validator_list_keys] +# vl.ripple.com +ED2677ABFFD1B33AC6FBC3062B71F1E8397C1505E1C42C64D11AD1B28FF73F4734 +# unl.xrplf.org +ED42AEC58B701EEBB77356FFFEC26F83C1F0407263530F068C7C73D392C7E06FD1 diff --git a/docker/telemetry/workload/README.md b/docker/telemetry/workload/README.md new file mode 100644 index 00000000000..eb926af6b9c --- /dev/null +++ b/docker/telemetry/workload/README.md @@ -0,0 +1,371 @@ +# Telemetry Workload Tools + +Synthetic workload generation and validation tools for xrpld's OpenTelemetry telemetry stack. These tools validate that all spans, metrics, dashboards, and log-trace correlation work end-to-end under controlled load. + +## Quick Start + +```bash +# Build xrpld with telemetry enabled +conan install . --build=missing -o telemetry=True +cmake --preset default -Dtelemetry=ON +cmake --build --preset default + +# Run full validation (starts everything, runs load, validates) +docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld + +# Cleanup when done +docker/telemetry/workload/run-full-validation.sh --cleanup +``` + +## Architecture + +The validation suite runs a multi-node xrpld cluster as local processes alongside +a Docker Compose telemetry stack. The cluster exercises consensus, peer-to-peer +spans (proposals, validations), and all metric pipelines. + +``` +run-full-validation.sh (shell orchestrator) + | + |-- docker-compose.workload.yaml + | |-- otel-collector (traces via OTLP + StatsD receiver) + | |-- tempo (trace backend + TraceQL search API) + | |-- prometheus (metrics scraping) + | |-- grafana (dashboards, provisioned automatically) + | + |-- generate-validator-keys.sh + | -> validator-keys.json, validators.txt + | + |-- Nx xrpld nodes (local processes, full telemetry) + | - Each node: [telemetry] enabled=1, trace_rpc/consensus/transactions + | - [signing_support] true (server-side signing for tx_submitter) + | - Peer discovery via [ips] (not [ips_fixed]) for active peer counts + | + |-- workload_orchestrator.py (phased load execution) + | |-- rpc_load_generator.py (WebSocket RPC traffic) + | |-- tx_submitter.py (transaction diversity) + | -> workload-report.json + per-phase reports + | + |-- validate_telemetry.py (pass/fail checks) + | -> validation-report.json + | + |-- benchmark.sh (baseline vs telemetry comparison) + -> benchmark-report-*.md +``` + +## Workload Profiles + +The workload orchestrator (`workload_orchestrator.py`) reads named profiles +from `workload-profiles.json` and executes sequential load phases. Within +each phase, the RPC generator and TX submitter run concurrently. + +### Available Profiles + +| Profile | Phases | Duration | Purpose | +| ----------------- | ------ | ---------------------------- | ----------------------------------------------------------- | +| `full-validation` | 6 | ~5 min + 1 min propagation | Full 18-dashboard coverage with burst/idle/plateau patterns | +| `quick-smoke` | 1 | ~30s + 30s propagation | Fast CI smoke test | +| `stress` | 3 | ~3.5 min + 1 min propagation | Heavy sustained load for benchmarking | + +### full-validation Phases + +| Phase | RPC Rate | TX TPS | Duration | Dashboard Coverage | +| ------------ | -------- | ------ | -------- | ----------------------------------------------- | +| warmup | 5 RPS | — | 30s | Node Health, Validator Health (baseline gauges) | +| steady-state | 30 RPS | 3 TPS | 60s | All dashboards (plateau data) | +| rpc-burst | 100 RPS | — | 30s | Job Queue, RPC Performance (latency spikes) | +| tx-flood | 5 RPS | 20 TPS | 30s | Fee Market & TxQ, Transaction Overview | +| mixed-peak | 50 RPS | 10 TPS | 60s | Consensus Health, Ledger Operations | +| cooldown | 5 RPS | — | 30s | Recovery patterns, state transitions | + +### Custom Profiles + +Add profiles to `workload-profiles.json`: + +```json +{ + "profiles": { + "my-custom": { + "description": "Custom profile for specific testing", + "phases": [ + { + "name": "phase-name", + "description": "What this phase exercises", + "duration_sec": 60, + "rpc": { "rate": 50, "weights": { "server_info": 80, "fee": 20 } }, + "tx": { "tps": 5, "weights": { "Payment": 100 } } + } + ], + "propagation_wait_sec": 30 + } + } +} +``` + +Set `"rpc"` or `"tx"` to `null` to skip that generator for a phase. +Custom `"weights"` override the default command/transaction distribution. + +## Tools Reference + +### run-full-validation.sh + +Orchestrates the complete validation pipeline. Starts the telemetry stack, starts a multi-node xrpld cluster, generates load, and validates the results. + +```bash +# Full validation with defaults (uses full-validation profile) +./run-full-validation.sh --xrpld /path/to/xrpld + +# Quick smoke test +./run-full-validation.sh --xrpld /path/to/xrpld --profile quick-smoke + +# Stress test with benchmarks +./run-full-validation.sh --xrpld /path/to/xrpld --profile stress --with-benchmark + +# Skip Loki checks (if Phase 8 not deployed) +./run-full-validation.sh --xrpld /path/to/xrpld --skip-loki +``` + +### workload_orchestrator.py + +Reads a named profile from `workload-profiles.json` and executes sequential +load phases. Within each phase, `rpc_load_generator.py` and `tx_submitter.py` +run as concurrent subprocesses. Produces per-phase reports and a combined +summary. + +```bash +# Run with a specific profile +python3 workload_orchestrator.py --profile full-validation + +# Multiple endpoints +python3 workload_orchestrator.py --profile full-validation \ + --endpoints ws://localhost:6006 ws://localhost:6007 + +# Save combined report +python3 workload_orchestrator.py --profile stress --report /tmp/report.json +``` + +### rpc_load_generator.py + +Generates RPC traffic matching realistic production distribution. Uses +xrpld's **native WebSocket command format** (`{"command": ...}`) with flat +parameters — the same format as `tx_submitter.py`. + +- 40% health checks (server_info, fee) +- 30% wallet queries (account_info, account_lines, account_objects) +- 15% explorer queries (ledger, ledger_data) +- 10% transaction lookups (tx, account_tx) +- 5% DEX queries (book_offers, amm_info) + +```bash +# Basic usage +python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120 + +# Multiple endpoints (round-robin) +python3 rpc_load_generator.py \ + --endpoints ws://localhost:6006 ws://localhost:6007 \ + --rate 100 --duration 300 + +# Custom weights +python3 rpc_load_generator.py --endpoints ws://localhost:6006 \ + --weights '{"server_info": 80, "account_info": 20}' +``` + +### tx_submitter.py + +Submits diverse transaction types to exercise the full span and metric surface. +Uses xrpld's **native WebSocket command format** (`{"command": ...}`) rather +than JSON-RPC format. The response payload is inside the `"result"` key, with +`"status"` at the top level. + +Supported transaction types: + +- Payment (XRP transfers) — exercises `tx.process`, `tx.receive`, `tx.apply` +- OfferCreate / OfferCancel (DEX activity) +- TrustSet (trust line creation) +- NFTokenMint / NFTokenCreateOffer (NFT activity) +- EscrowCreate / EscrowFinish (escrow lifecycle) +- AMMCreate / AMMDeposit (AMM pool operations) + +Requires `[signing_support] true` in the node config for server-side signing. + +```bash +# Basic usage +python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120 + +# Custom mix +python3 tx_submitter.py --endpoint ws://localhost:6006 \ + --weights '{"Payment": 60, "OfferCreate": 20, "TrustSet": 20}' +``` + +### validate_telemetry.py + +Automated validation that all expected telemetry data exists. Every metric and span is required — if it doesn't fire, the validation fails. + +- **Span validation**: All span types from `expected_spans.json` with required attributes and parent-child hierarchies +- **Metric validation**: All metrics from `expected_metrics.json` — SpanMetrics, StatsD gauges/counters/histograms, Phase 9 OTLP metrics. Every listed metric must have > 0 series. Uses the Prometheus `/api/v1/series` endpoint (not instant queries) to avoid false negatives from stale gauges. +- **Log-trace correlation**: trace_id/span_id in Loki logs (requires Loki) +- **Dashboard validation**: All 10 Grafana dashboards load with panels + +```bash +# Run all validations +python3 validate_telemetry.py --report /tmp/report.json + +# Skip Loki checks +python3 validate_telemetry.py --skip-loki --report /tmp/report.json +``` + +### OTel Timings Regression Gate + +`capture_timings.py` + `compare_to_baseline.py` implement a regression gate +that compares OTel-derived per-span/per-RPC/per-job timings against a +committed baseline. Unlike `benchmark.sh` (which measures the overhead of +enabling telemetry on the current binary), this gate catches **xrpld +performance regressions over time** by diffing against a stored baseline +from a prior run. + +How it runs inside the validation pipeline: + +1. `run-full-validation.sh` executes the normal workload and validation suite. +2. After validation, `capture_timings.py` queries Prometheus for every + metric in `regression-metrics.json` and writes `reports/timings.json`. +3. `compare_to_baseline.py` reads `timings.json`, + `baselines/baseline-timings.json`, and `regression-thresholds.json`, + then either: + - Prints the paste-me JSON block (when the baseline is a placeholder + or empty) and exits 0. + - Prints a delta table, writes `reports/regression-report.json`, and + exits non-zero if any metric breached both the percentage AND + absolute bound. + +Bootstrapping a baseline: + +1. Push the branch. The `Telemetry Validation` CI run prints the full + timings JSON under "Paste into `baselines/baseline-timings.json`" in + the workflow Step Summary. +2. Open a PR copying that JSON block verbatim into + `baselines/baseline-timings.json`. Reviewer approval is the audit gate. +3. Subsequent runs compare against it; the gate fails on regression. + +Per-run tuning: + +- `--skip-regression` disables the gate (local exploration only). +- `REGRESSION_WINDOW` env var overrides the default Prometheus `rate()` + window (`3m`). Keep close to the workload duration. +- Metric surface lives in `regression-metrics.json`; thresholds in + `regression-thresholds.json`; both are reviewed changes. + +See [`baselines/README.md`](./baselines/README.md) for the baseline +lifecycle and refresh process. + +### benchmark.sh + +Compares baseline (no telemetry) vs telemetry-enabled performance: + +```bash +./benchmark.sh --xrpld /path/to/xrpld --duration 300 +``` + +Thresholds (configurable via environment): + +| Metric | Threshold | Env Variable | +| ----------------- | --------- | --------------------------- | +| CPU overhead | < 3% | BENCH_CPU_OVERHEAD_PCT | +| Memory overhead | < 5MB | BENCH_MEM_OVERHEAD_MB | +| RPC p99 latency | < 2ms | BENCH_RPC_LATENCY_IMPACT_MS | +| Throughput impact | < 5% | BENCH_TPS_IMPACT_PCT | +| Consensus impact | < 1% | BENCH_CONSENSUS_IMPACT_PCT | + +## Reading Validation Reports + +The validation report (`validation-report.json`) is structured as: + +```json +{ + "summary": { + "total": 45, + "passed": 42, + "failed": 3, + "all_passed": false + }, + "checks": [ + { + "name": "span.rpc.ws_message", + "category": "span", + "passed": true, + "message": "rpc.ws_message: 15 traces found", + "details": { "trace_count": 15 } + } + ] +} +``` + +Categories: + +- **span**: Span type existence and attribute validation +- **metric**: Prometheus metric existence +- **log**: Log-trace correlation checks +- **dashboard**: Grafana dashboard accessibility + +## CI Integration + +The validation runs as a GitHub Actions workflow (`.github/workflows/telemetry-validation.yml`): + +- Triggered manually or on pushes to telemetry branches +- Builds xrpld, starts the full stack, runs load, validates +- Uploads reports as artifacts +- Posts summary to PR + +## Configuration Files + +| File | Purpose | +| --------------------------------- | ------------------------------------------------------------- | +| `workload-profiles.json` | Named load profiles with phase definitions | +| `expected_spans.json` | Span inventory (names, attributes, hierarchies, config flags) | +| `expected_metrics.json` | Metric inventory — every listed metric must be present | +| `test_accounts.json` | Test account roles (keys generated at runtime) | +| `regression-metrics.json` | Metric surface for the OTel regression gate | +| `regression-thresholds.json` | Per-metric regression bounds (pct AND abs) | +| `baselines/baseline-timings.json` | Committed baseline — populated from first CI run | +| `requirements.txt` | Python dependencies | + +### expected_metrics.json Format + +```json +{ + "category_name": { + "description": "Human-readable description.", + "metrics": ["metric_1", "metric_2"] + } +} +``` + +Every metric listed must produce > 0 Prometheus series during the validation run. If a metric doesn't fire, the workload generators need to produce enough load to trigger it. + +### expected_spans.json Format + +Each span entry defines its name, category, parent (for hierarchy validation), +required attributes, and the `config_flag` that must be enabled: + +```json +{ + "name": "rpc.command.*", + "category": "rpc", + "parent": "rpc.process", + "required_attributes": ["command", "version", "rpc_role", "rpc_status"], + "config_flag": "trace_rpc" +} +``` + +## Node Configuration Notes + +The orchestrator (`run-full-validation.sh`) generates node configs with: + +- `[telemetry] enabled=1` with all trace categories (`trace_rpc`, `trace_consensus`, `trace_transactions`) +- `[signing_support] true` — required for `tx_submitter.py` to submit signed transactions via WebSocket +- `[ips]` (not `[ips_fixed]`) — ensures peer connections are counted in `Peer_Finder_Active_Inbound/Outbound_Peers` metrics (fixed peers are excluded from these counters by design) + +## StatsD Gauge Behaviour + +Beast::insight StatsD gauges only emit when their value _changes_ from the previous sample. This can cause two problems in the validation environment: + +1. **Initial-zero gauges** — if a gauge value is 0 from startup and never changes, the gauge would never emit. To address this, `StatsDGaugeImpl` initializes `m_dirty = true`, ensuring the first flush always emits the initial value. +2. **Stale gauges** — once a gauge stabilizes (e.g., peer count stays at 1), it stops emitting new data points. Prometheus marks it stale after ~5 minutes. The validation script uses the Prometheus `/api/v1/series` endpoint instead of instant queries to catch such gauges. diff --git a/docker/telemetry/workload/baselines/README.md b/docker/telemetry/workload/baselines/README.md new file mode 100644 index 00000000000..515a3f561fe --- /dev/null +++ b/docker/telemetry/workload/baselines/README.md @@ -0,0 +1,72 @@ +# Performance Baselines + +This directory holds the committed baseline file used by the OTel-driven regression gate. + +## How the gate works + +After the validation suite runs, `capture_timings.py` queries Prometheus for the timings +declared in [`../regression-metrics.json`](../regression-metrics.json) and writes a +`timings.json`. Then `compare_to_baseline.py` reads [`baseline-timings.json`](./baseline-timings.json), +[`../regression-thresholds.json`](../regression-thresholds.json), and the captured +`timings.json`. The comparator picks one of two modes automatically: + +- **Placeholder baseline** (`"placeholder": true` or empty `metrics`): the comparator + prints the captured timings JSON in exactly the format expected for this file, then + exits 0 without gating. This is how we bootstrap the baseline. +- **Populated baseline**: the comparator diffs per-metric, enforces the thresholds + (regression = current exceeds baseline on BOTH the percentage AND absolute bound), + and exits non-zero on any regression. + +The regression gate runs against whatever workload profile `run-full-validation.sh` +was invoked with. Capture and comparison are profile-agnostic — they only read +Prometheus — so all existing profiles (`full-validation`, `quick-smoke`, `stress`) +continue to work unchanged. + +## Bootstrapping the baseline + +1. Merge a CI run with a `"placeholder": true` baseline. The telemetry-validation + workflow runs, fails no gate, and prints the captured timings block to the workflow + Step Summary under the heading `### Paste into baselines/baseline-timings.json`. +2. Open a new PR. Copy the full JSON block from the Step Summary (or download the + `timings.json` artifact) into this file, replacing the placeholder contents. The + JSON is emitted in the exact byte-for-byte format this file expects — sorted keys, + 2-space indent, trailing newline. +3. The committed baseline PR needs reviewer approval just like any other code change. + This is the primary audit point for "who moved the performance bar." + +## Refreshing the baseline + +Refresh when a legitimate performance change lands on `develop` (for example, a +deliberate rewrite that changes a span's structure). The process is identical to +bootstrapping: run CI with the current baseline, inspect the delta, and if the +new numbers should become the norm, open a PR pasting the fresh timings into +`baseline-timings.json`. The reviewer decides whether the new baseline is acceptable. + +Do **not** edit `baseline-timings.json` by hand outside of this process — every entry +should trace back to a real CI run so variance characteristics are preserved. + +## Schema + +```json +{ + "schema_version": 1, + "captured_at": "2026-04-24T17:30:00Z", + "window": "3m", + "git_sha": "", + "profile": "", + "metrics": { + "span.tx.process.p99": { "value": 12.4, "unit": "ms" }, + "rpc.server_info.p95": { "value": 850.0, "unit": "us" }, + "job.transaction.queued.p95": { "value": 1500.0, "unit": "us" } + } +} +``` + +Placeholder baselines additionally include `"placeholder": true`. The comparator +detects this field (or an empty `metrics` object) to switch into "populate" mode +instead of enforcing thresholds. Remove the `placeholder` key when pasting real +captured timings. + +Missing metrics (value `null`) in a captured run do not count as regressions — they +are reported separately in `regression-report.json` under `missing_in_current`. +This keeps the gate robust when a profile doesn't exercise every span on every run. diff --git a/docker/telemetry/workload/baselines/baseline-timings.json b/docker/telemetry/workload/baselines/baseline-timings.json new file mode 100644 index 00000000000..cafd683e3c3 --- /dev/null +++ b/docker/telemetry/workload/baselines/baseline-timings.json @@ -0,0 +1,133 @@ +{ + "captured_at": "2026-06-05T18:41:52Z", + "git_sha": "fd1c8c6060f7a15cc9e65b16f99629d9ab7ac7dc", + "metrics": { + "job.acceptLedger.queued.p95": { + "unit": "us", + "value": 96.78571428571428 + }, + "job.acceptLedger.running.p95": { + "unit": "us", + "value": 10562.499999999945 + }, + "job.transaction.queued.p95": { + "unit": "us", + "value": 478.96551724137925 + }, + "job.transaction.running.p95": { + "unit": "us", + "value": 494.1361256544502 + }, + "span.consensus.accept.p50": { + "unit": "ms", + "value": 1.059405940594059 + }, + "span.consensus.accept.p95": { + "unit": "ms", + "value": 9.749999999999996 + }, + "span.consensus.accept.p99": { + "unit": "ms", + "value": 23.704545454545432 + }, + "span.consensus.ledger_close.p50": { + "unit": "ms", + "value": 0.5284697508896797 + }, + "span.consensus.ledger_close.p95": { + "unit": "ms", + "value": 1.511111111111103 + }, + "span.consensus.ledger_close.p99": { + "unit": "ms", + "value": 7.878571428571429 + }, + "span.ledger.build.p50": { + "unit": "ms", + "value": 0.7412060301507538 + }, + "span.ledger.build.p95": { + "unit": "ms", + "value": 4.611111111111112 + }, + "span.ledger.build.p99": { + "unit": "ms", + "value": 7.541666666666674 + }, + "span.ledger.store.p50": { + "unit": "ms", + "value": 0.5 + }, + "span.ledger.store.p95": { + "unit": "ms", + "value": 0.95 + }, + "span.ledger.store.p99": { + "unit": "ms", + "value": 0.9900000000000001 + }, + "span.ledger.validate.p50": { + "unit": "ms", + "value": 0.5283687943262412 + }, + "span.ledger.validate.p95": { + "unit": "ms", + "value": 1.3666666666666627 + }, + "span.ledger.validate.p99": { + "unit": "ms", + "value": 6.699999999999978 + }, + "span.rpc.process.p50": { + "unit": "ms", + "value": null + }, + "span.rpc.process.p95": { + "unit": "ms", + "value": null + }, + "span.rpc.process.p99": { + "unit": "ms", + "value": null + }, + "span.rpc.ws_message.p50": { + "unit": "ms", + "value": 0.5026522773001647 + }, + "span.rpc.ws_message.p95": { + "unit": "ms", + "value": 0.9550393268703128 + }, + "span.rpc.ws_message.p99": { + "unit": "ms", + "value": 0.9952515090543261 + }, + "span.tx.apply.p50": { + "unit": "ms", + "value": 0.6330472103004292 + }, + "span.tx.apply.p95": { + "unit": "ms", + "value": 4.203389830508474 + }, + "span.tx.apply.p99": { + "unit": "ms", + "value": 5.083333333333319 + }, + "span.tx.process.p50": { + "unit": "ms", + "value": 0.5042801992591597 + }, + "span.tx.process.p95": { + "unit": "ms", + "value": 0.9581323781882418 + }, + "span.tx.process.p99": { + "unit": "ms", + "value": 0.998474791584883 + } + }, + "profile": "full-validation", + "schema_version": 1, + "window": "3m" +} diff --git a/docker/telemetry/workload/benchmark-results/.gitkeep b/docker/telemetry/workload/benchmark-results/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docker/telemetry/workload/benchmark.sh b/docker/telemetry/workload/benchmark.sh new file mode 100755 index 00000000000..954322f7143 --- /dev/null +++ b/docker/telemetry/workload/benchmark.sh @@ -0,0 +1,396 @@ +#!/usr/bin/env bash +# benchmark.sh — Performance benchmark for rippled telemetry overhead. +# +# Runs two identical workloads against a rippled cluster: +# 1. Baseline: telemetry disabled ([telemetry] enabled=0) +# 2. Telemetry: full telemetry enabled (traces + StatsD + all categories) +# +# Compares CPU, memory, RPC latency, TPS, and consensus round time. +# Outputs a Markdown table with pass/fail against configured thresholds. +# +# Usage: +# ./benchmark.sh --xrpld /path/to/xrpld --duration 300 +# +# Thresholds (configurable via environment variables): +# BENCH_CPU_OVERHEAD_PCT=3 CPU overhead < 3% +# BENCH_MEM_OVERHEAD_MB=5 Memory overhead < 5MB +# BENCH_RPC_LATENCY_IMPACT_MS=2 RPC p99 latency impact < 2ms +# BENCH_TPS_IMPACT_PCT=5 Throughput impact < 5% +# BENCH_CONSENSUS_IMPACT_PCT=1 Consensus round time impact < 1% + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colored output helpers +# --------------------------------------------------------------------------- +log() { printf "\033[1;34m[BENCH]\033[0m %s\n" "$*"; } +ok() { printf "\033[1;32m[BENCH]\033[0m %s\n" "$*"; } +warn() { printf "\033[1;33m[BENCH]\033[0m %s\n" "$*"; } +fail() { printf "\033[1;31m[BENCH]\033[0m %s\n" "$*"; } +die() { + printf "\033[1;31m[BENCH]\033[0m %s\n" "$*" >&2 + exit 1 +} + +# --------------------------------------------------------------------------- +# Defaults and thresholds +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +# Configurable thresholds via environment variables. +CPU_THRESHOLD="${BENCH_CPU_OVERHEAD_PCT:-3}" +MEM_THRESHOLD="${BENCH_MEM_OVERHEAD_MB:-5}" +RPC_THRESHOLD="${BENCH_RPC_LATENCY_IMPACT_MS:-2}" +TPS_THRESHOLD="${BENCH_TPS_IMPACT_PCT:-5}" +CONSENSUS_THRESHOLD="${BENCH_CONSENSUS_IMPACT_PCT:-1}" + +XRPLD="${BENCH_XRPLD:-$REPO_ROOT/.build/xrpld}" +DURATION=300 +NUM_NODES=3 +WORKDIR="/tmp/xrpld-benchmark" +RESULTS_DIR="$SCRIPT_DIR/benchmark-results" +RPC_PORT_BASE=5020 +PEER_PORT_BASE=51250 + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +usage() { + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --xrpld PATH Path to xrpld binary (default: \$REPO_ROOT/.build/xrpld)" + echo " --duration SECS Benchmark duration per run (default: 300)" + echo " --nodes NUM Number of validator nodes (default: 3)" + echo " --output DIR Results output directory" + echo " -h, --help Show this help" + exit 0 +} + +while [ $# -gt 0 ]; do + case "$1" in + --xrpld) + XRPLD="$2" + shift 2 + ;; + --duration) + DURATION="$2" + shift 2 + ;; + --nodes) + NUM_NODES="$2" + shift 2 + ;; + --output) + RESULTS_DIR="$2" + shift 2 + ;; + -h | --help) usage ;; + *) die "Unknown option: $1" ;; + esac +done + +# Validate prerequisites. +[ -x "$XRPLD" ] || die "xrpld not found at $XRPLD" +command -v jq >/dev/null 2>&1 || die "jq not found" +command -v bc >/dev/null 2>&1 || die "bc not found" +command -v curl >/dev/null 2>&1 || die "curl not found" + +mkdir -p "$RESULTS_DIR" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) + +# --------------------------------------------------------------------------- +# Node cluster management +# --------------------------------------------------------------------------- +start_cluster() { + local telemetry_enabled="$1" + local label="$2" + + log "Starting $NUM_NODES-node cluster ($label, telemetry=$telemetry_enabled)..." + + rm -rf "$WORKDIR" + mkdir -p "$WORKDIR" + + # Generate keys using first node. + bash "$SCRIPT_DIR/generate-validator-keys.sh" "$XRPLD" "$NUM_NODES" "$WORKDIR" + + # Build per-node configs. + for i in $(seq 1 "$NUM_NODES"); do + local node_dir="$WORKDIR/node$i" + mkdir -p "$node_dir/nudb" "$node_dir/db" + + local rpc_port + rpc_port=$((RPC_PORT_BASE + i - 1)) + local peer_port + peer_port=$((PEER_PORT_BASE + i - 1)) + local seed + seed=$(jq -r ".[$((i - 1))].seed" "$WORKDIR/validator-keys.json") + + # Build ips_fixed list. + local ips_fixed="" + for j in $(seq 1 "$NUM_NODES"); do + if [ "$j" -ne "$i" ]; then + ips_fixed="${ips_fixed}127.0.0.1 $((PEER_PORT_BASE + j - 1)) +" + fi + done + + # Build telemetry section. + local telemetry_section="" + if [ "$telemetry_enabled" = "1" ]; then + telemetry_section=" +[telemetry] +enabled=1 +service_instance_id=bench-node-${i} +endpoint=http://localhost:4318/v1/traces +exporter=otlp_http +batch_size=512 +batch_delay_ms=2000 +max_queue_size=2048 +trace_rpc=1 +trace_transactions=1 +trace_consensus=1 +trace_peer=1 +trace_ledger=1 + +[insight] +server=otel +endpoint=http://localhost:4318/v1/metrics +prefix=xrpld" + else + telemetry_section=" +[telemetry] +enabled=0" + fi + + cat >"$node_dir/xrpld.cfg" <"$node_dir/stdout.log" 2>&1 & + echo $! >"$node_dir/xrpld.pid" + done + + # Wait for consensus. + log "Waiting for consensus..." + for attempt in $(seq 1 120); do + local ready=0 + for i in $(seq 1 "$NUM_NODES"); do + local port + port=$((RPC_PORT_BASE + i - 1)) + local state + state=$(curl -sf "http://localhost:$port" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.server_state' 2>/dev/null || echo "") + if [ "$state" = "proposing" ]; then + ready=$((ready + 1)) + fi + done + if [ "$ready" -ge "$NUM_NODES" ]; then + ok "All $NUM_NODES nodes proposing (attempt $attempt)" + break + fi + if [ "$attempt" -eq 120 ]; then + warn "Consensus timeout — $ready/$NUM_NODES nodes ready" + fi + sleep 1 + done + + # Let the cluster stabilize. + sleep 5 +} + +stop_cluster() { + log "Stopping cluster..." + for i in $(seq 1 "$NUM_NODES"); do + local pidfile="$WORKDIR/node$i/xrpld.pid" + if [ -f "$pidfile" ]; then + kill "$(cat "$pidfile")" 2>/dev/null || true + fi + done + pkill -f "$WORKDIR" 2>/dev/null || true + sleep 3 +} + +# Build RPC ports CSV string. +rpc_ports_csv() { + local ports="" + for i in $(seq 1 "$NUM_NODES"); do + [ -n "$ports" ] && ports="$ports," + ports="$ports$((RPC_PORT_BASE + i - 1))" + done + echo "$ports" +} + +# --------------------------------------------------------------------------- +# Run benchmark +# --------------------------------------------------------------------------- +log "=" +log " rippled Telemetry Performance Benchmark" +log " Nodes: $NUM_NODES | Duration: ${DURATION}s | Binary: $XRPLD" +log "=" + +# --- Baseline run --- +BASELINE_FILE="$RESULTS_DIR/baseline-${TIMESTAMP}.json" +start_cluster "0" "baseline" +bash "$SCRIPT_DIR/collect_system_metrics.sh" "$(rpc_ports_csv)" "$DURATION" "$BASELINE_FILE" +stop_cluster + +# --- Telemetry run --- +TELEMETRY_FILE="$RESULTS_DIR/telemetry-${TIMESTAMP}.json" +start_cluster "1" "telemetry" +bash "$SCRIPT_DIR/collect_system_metrics.sh" "$(rpc_ports_csv)" "$DURATION" "$TELEMETRY_FILE" +stop_cluster + +# --------------------------------------------------------------------------- +# Compare results +# --------------------------------------------------------------------------- +log "Comparing results..." + +read_metric() { + local file="$1" + local key="$2" + jq -r ".$key // 0" "$file" +} + +BASE_CPU=$(read_metric "$BASELINE_FILE" "cpu_pct_avg") +TELE_CPU=$(read_metric "$TELEMETRY_FILE" "cpu_pct_avg") +CPU_DELTA=$(echo "scale=2; $TELE_CPU - $BASE_CPU" | bc 2>/dev/null || echo "0") + +BASE_MEM=$(read_metric "$BASELINE_FILE" "memory_rss_mb_peak") +TELE_MEM=$(read_metric "$TELEMETRY_FILE" "memory_rss_mb_peak") +MEM_DELTA=$(echo "scale=2; $TELE_MEM - $BASE_MEM" | bc 2>/dev/null || echo "0") + +BASE_RPC=$(read_metric "$BASELINE_FILE" "rpc_p99_ms") +TELE_RPC=$(read_metric "$TELEMETRY_FILE" "rpc_p99_ms") +RPC_DELTA=$(echo "scale=2; $TELE_RPC - $BASE_RPC" | bc 2>/dev/null || echo "0") + +BASE_TPS=$(read_metric "$BASELINE_FILE" "tps") +TELE_TPS=$(read_metric "$TELEMETRY_FILE" "tps") +if [[ "$(echo "$BASE_TPS > 0" | bc 2>/dev/null)" = "1" ]]; then + TPS_IMPACT=$(echo "scale=2; ($BASE_TPS - $TELE_TPS) / $BASE_TPS * 100" | bc 2>/dev/null || echo "0") +else + TPS_IMPACT="0" +fi + +BASE_CONS=$(read_metric "$BASELINE_FILE" "consensus_round_p95_ms") +TELE_CONS=$(read_metric "$TELEMETRY_FILE" "consensus_round_p95_ms") +if [[ "$(echo "$BASE_CONS > 0" | bc 2>/dev/null)" = "1" ]]; then + CONS_IMPACT=$(echo "scale=2; ($TELE_CONS - $BASE_CONS) / $BASE_CONS * 100" | bc 2>/dev/null || echo "0") +else + CONS_IMPACT="0" +fi + +# --------------------------------------------------------------------------- +# Pass/fail checks +# --------------------------------------------------------------------------- +PASS_COUNT=0 +FAIL_COUNT=0 + +check_threshold() { + local name="$1" + local actual="$2" + local threshold="$3" + local unit="$4" + + # Compare: actual <= threshold + if [[ "$(echo "$actual <= $threshold" | bc 2>/dev/null)" = "1" ]]; then + ok "$name: ${actual}${unit} <= ${threshold}${unit} PASS" + PASS_COUNT=$((PASS_COUNT + 1)) + echo "PASS" + else + fail "$name: ${actual}${unit} > ${threshold}${unit} FAIL" + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo "FAIL" + fi +} + +CPU_RESULT=$(check_threshold "CPU overhead" "$CPU_DELTA" "$CPU_THRESHOLD" "%") +MEM_RESULT=$(check_threshold "Memory overhead" "$MEM_DELTA" "$MEM_THRESHOLD" "MB") +RPC_RESULT=$(check_threshold "RPC p99 impact" "$RPC_DELTA" "$RPC_THRESHOLD" "ms") +TPS_RESULT=$(check_threshold "TPS impact" "$TPS_IMPACT" "$TPS_THRESHOLD" "%") +CONS_RESULT=$(check_threshold "Consensus impact" "$CONS_IMPACT" "$CONSENSUS_THRESHOLD" "%") + +# --------------------------------------------------------------------------- +# Output Markdown table +# --------------------------------------------------------------------------- +REPORT_FILE="$RESULTS_DIR/benchmark-report-${TIMESTAMP}.md" + +cat >"$REPORT_FILE" <", + "profile": "regression", + "metrics": { + "span.tx.process.p99": {"value": 12.4, "unit": "ms"}, + "rpc.server_info.p95": {"value": 850.0, "unit": "us"}, + ... + } + } + +Usage:: + + python3 capture_timings.py \\ + --prometheus http://localhost:9090 \\ + --metrics regression-metrics.json \\ + --output /tmp/timings.json \\ + --window 3m \\ + --profile regression +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +import aiohttp + +from prom_queries import build_query_plan, run_query_plan + +logger = logging.getLogger("capture_timings") + +SCHEMA_VERSION = 1 + + +async def capture( + prom_url: str, + metrics_path: Path, + window: str, + profile: str, +) -> dict: + """Build and execute the query plan, return the full report dict.""" + plan = build_query_plan(metrics_path, window=window) + logger.info("Capturing %d metrics from %s (window=%s)", len(plan), prom_url, window) + + async with aiohttp.ClientSession() as session: + metrics = await run_query_plan(session, prom_url, plan) + + return { + "schema_version": SCHEMA_VERSION, + "captured_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "window": window, + "git_sha": _detect_git_sha(), + "profile": profile, + "metrics": dict(sorted(metrics.items())), + } + + +def _detect_git_sha() -> str: + """Return the current commit SHA from env or git, else ``"unknown"``. + + Prefers ``GITHUB_SHA`` (set in Actions), falls back to ``git rev-parse``. + Silent fallback is fine here — a missing SHA only affects the captured + metadata, not the comparison logic. + """ + env_sha = os.environ.get("GITHUB_SHA") + if env_sha: + return env_sha + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + if result.returncode == 0: + return result.stdout.strip() + except (OSError, subprocess.SubprocessError): + pass + return "unknown" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--prometheus", + default="http://localhost:9090", + help="Prometheus base URL (default: http://localhost:9090)", + ) + parser.add_argument( + "--metrics", + type=Path, + default=Path(__file__).parent / "regression-metrics.json", + help="Path to regression-metrics.json", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="Where to write the captured timings JSON", + ) + parser.add_argument( + "--window", + default="3m", + help="Prometheus rate() window (default: 3m)", + ) + parser.add_argument( + "--profile", + default="regression", + help="Workload profile used during capture (metadata only)", + ) + parser.add_argument( + "--min-capture-ratio", + type=float, + default=0.5, + help="Fail if fewer than this fraction of metrics are captured (default: 0.5)", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable debug logging", + ) + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s %(name)s: %(message)s", + ) + + report = asyncio.run( + capture( + prom_url=args.prometheus, + metrics_path=args.metrics, + window=args.window, + profile=args.profile, + ) + ) + + args.output.parent.mkdir(parents=True, exist_ok=True) + with open(args.output, "w") as f: + json.dump(report, f, indent=2, sort_keys=True) + f.write("\n") + + captured = sum(1 for v in report["metrics"].values() if v["value"] is not None) + total = len(report["metrics"]) + logger.info("Wrote %s (%d/%d metrics captured)", args.output, captured, total) + + if total > 0 and (captured / total) < args.min_capture_ratio: + logger.error( + "Only %d/%d (%.0f%%) metrics captured — below the %.0f%% minimum. " + "Is Prometheus reachable at %s?", + captured, + total, + captured / total * 100, + args.min_capture_ratio * 100, + args.prometheus, + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/telemetry/workload/collect_system_metrics.sh b/docker/telemetry/workload/collect_system_metrics.sh new file mode 100755 index 00000000000..58c1c7b8961 --- /dev/null +++ b/docker/telemetry/workload/collect_system_metrics.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +# collect_system_metrics.sh — Collect CPU, memory, and RPC latency metrics +# from running xrpld nodes for benchmark comparison. +# +# Samples system metrics at regular intervals and writes a JSON summary. +# Used by benchmark.sh for baseline vs telemetry comparison. +# +# Usage: +# ./collect_system_metrics.sh +# +# Example: +# ./collect_system_metrics.sh "5005,5006,5007" 300 /tmp/metrics-baseline.json +# +# Output JSON format: +# { +# "cpu_pct_avg": 12.5, +# "memory_rss_mb_peak": 450.2, +# "rpc_p99_ms": 15.3, +# "tps": 4.8, +# "consensus_round_p95_ms": 3200, +# "samples": 60 +# } + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colored output helpers +# --------------------------------------------------------------------------- +log() { printf "\033[1;34m[METRICS]\033[0m %s\n" "$*"; } +ok() { printf "\033[1;32m[METRICS]\033[0m %s\n" "$*"; } +die() { + printf "\033[1;31m[METRICS]\033[0m %s\n" "$*" >&2 + exit 1 +} + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +usage() { + echo "Usage: $0 " + echo "" + echo "Arguments:" + echo " rpc_ports_csv Comma-separated RPC ports (e.g., 5005,5006,5007)" + echo " duration_seconds How long to collect metrics" + echo " output_file Path to write JSON results" + exit 1 +} + +if [ $# -lt 3 ]; then + usage +fi + +RPC_PORTS_CSV="$1" +DURATION="$2" +OUTPUT_FILE="$3" + +IFS=',' read -ra RPC_PORTS <<<"$RPC_PORTS_CSV" +SAMPLE_INTERVAL=5 +SAMPLES=$((DURATION / SAMPLE_INTERVAL)) + +log "Collecting metrics for ${DURATION}s (${SAMPLES} samples, ${#RPC_PORTS[@]} nodes)..." + +# --------------------------------------------------------------------------- +# Temporary files for aggregation +# --------------------------------------------------------------------------- +TMPDIR_METRICS="$(mktemp -d)" +CPU_FILE="$TMPDIR_METRICS/cpu.txt" +MEM_FILE="$TMPDIR_METRICS/mem.txt" +RPC_FILE="$TMPDIR_METRICS/rpc.txt" +LEDGER_FILE="$TMPDIR_METRICS/ledger.txt" + +touch "$CPU_FILE" "$MEM_FILE" "$RPC_FILE" "$LEDGER_FILE" + +cleanup() { + rm -rf "$TMPDIR_METRICS" +} +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# Get initial ledger sequence for TPS calculation +# --------------------------------------------------------------------------- +INITIAL_SEQ=0 +INITIAL_TIME=$(date +%s) +for port in "${RPC_PORTS[@]}"; do + seq=$(curl -sf "http://localhost:$port" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) + if [ "$seq" -gt "$INITIAL_SEQ" ]; then + INITIAL_SEQ=$seq + fi +done +log "Initial validated ledger seq: $INITIAL_SEQ" + +# --------------------------------------------------------------------------- +# Sampling loop +# --------------------------------------------------------------------------- +for sample in $(seq 1 "$SAMPLES"); do + # Collect CPU usage for xrpld processes. + # Uses ps to find all xrpld processes and average their CPU%. + cpu_sum=0 + cpu_count=0 + while IFS= read -r line; do + cpu_val=$(echo "$line" | awk '{print $1}') + if [ -n "$cpu_val" ] && [ "$cpu_val" != "0.0" ]; then + cpu_sum=$(echo "$cpu_sum + $cpu_val" | bc 2>/dev/null || echo "$cpu_sum") + cpu_count=$((cpu_count + 1)) + fi + done < <(ps aux 2>/dev/null | grep '[x]rpld' | awk '{print $3}') + + if [ "$cpu_count" -gt 0 ]; then + cpu_avg=$(echo "scale=2; $cpu_sum / $cpu_count" | bc 2>/dev/null || echo "0") + echo "$cpu_avg" >>"$CPU_FILE" + fi + + # Collect memory RSS for xrpld processes. + while IFS= read -r line; do + rss_kb=$(echo "$line" | awk '{print $1}') + if [ -n "$rss_kb" ] && [ "$rss_kb" != "0" ]; then + rss_mb=$(echo "scale=2; $rss_kb / 1024" | bc 2>/dev/null || echo "0") + echo "$rss_mb" >>"$MEM_FILE" + fi + done < <(ps aux 2>/dev/null | grep '[x]rpld' | awk '{print $6}') + + # Collect RPC latency from each node. + for port in "${RPC_PORTS[@]}"; do + start_ms=$(date +%s%N) + curl -sf "http://localhost:$port" \ + -d '{"method":"server_info"}' >/dev/null 2>&1 || true + end_ms=$(date +%s%N) + latency_ms=$(((end_ms - start_ms) / 1000000)) + echo "$latency_ms" >>"$RPC_FILE" + done + + # Record current validated ledger seq. + for port in "${RPC_PORTS[@]}"; do + seq=$(curl -sf "http://localhost:$port" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) + echo "$seq" >>"$LEDGER_FILE" + break # Only need one node's seq per sample. + done + + # Progress indicator. + if [ $((sample % 10)) -eq 0 ]; then + log " Sample $sample/$SAMPLES..." + fi + + sleep "$SAMPLE_INTERVAL" +done + +# --------------------------------------------------------------------------- +# Compute aggregated metrics +# --------------------------------------------------------------------------- +log "Computing aggregated metrics..." + +# CPU average. +if [ -s "$CPU_FILE" ]; then + CPU_AVG=$(awk '{ sum += $1; n++ } END { if (n>0) printf "%.2f", sum/n; else print "0" }' "$CPU_FILE") +else + CPU_AVG="0" +fi + +# Memory peak RSS (MB). +if [ -s "$MEM_FILE" ]; then + MEM_PEAK=$(sort -n "$MEM_FILE" | tail -1) +else + MEM_PEAK="0" +fi + +# RPC latency p99 (ms). +if [ -s "$RPC_FILE" ]; then + RPC_COUNT=$(wc -l <"$RPC_FILE") + P99_INDEX=$(echo "scale=0; $RPC_COUNT * 99 / 100" | bc) + RPC_P99=$(sort -n "$RPC_FILE" | sed -n "${P99_INDEX}p") + [ -z "$RPC_P99" ] && RPC_P99="0" +else + RPC_P99="0" +fi + +# TPS calculation from ledger sequence advancement. +FINAL_SEQ=0 +for port in "${RPC_PORTS[@]}"; do + seq=$(curl -sf "http://localhost:$port" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) + if [ "$seq" -gt "$FINAL_SEQ" ]; then + FINAL_SEQ=$seq + fi +done +FINAL_TIME=$(date +%s) +ELAPSED=$((FINAL_TIME - INITIAL_TIME)) +LEDGER_ADVANCE=$((FINAL_SEQ - INITIAL_SEQ)) +if [ "$ELAPSED" -gt 0 ] && [ "$LEDGER_ADVANCE" -gt 0 ]; then + # Rough TPS: assume ~avg_txs_per_ledger * ledgers / elapsed. + # Without tx count, use ledger close rate as proxy. + TPS=$(echo "scale=2; $LEDGER_ADVANCE / $ELAPSED" | bc 2>/dev/null || echo "0") +else + TPS="0" +fi + +# Consensus round time p95 (from ledger close interval). +# Approximate by looking at ledger sequence progression intervals. +if [ -s "$LEDGER_FILE" ]; then + # Calculate intervals between consecutive ledger sequences. + LEDGER_COUNT=$(wc -l <"$LEDGER_FILE") + # Rough estimate: DURATION / number_of_distinct_ledgers * 1000 ms + UNIQUE_LEDGERS=$(sort -u "$LEDGER_FILE" | wc -l) + if [ "$UNIQUE_LEDGERS" -gt 1 ]; then + CONSENSUS_P95=$(echo "scale=0; $DURATION * 1000 / ($UNIQUE_LEDGERS - 1)" | bc 2>/dev/null || echo "0") + else + CONSENSUS_P95="0" + fi +else + CONSENSUS_P95="0" +fi + +# --------------------------------------------------------------------------- +# Write output JSON +# --------------------------------------------------------------------------- +cat >"$OUTPUT_FILE" < dict: + with open(path) as f: + return json.load(f) + + +def is_placeholder(baseline: dict) -> bool: + """A baseline is a placeholder if explicitly marked OR metrics are empty.""" + if baseline.get("placeholder") is True: + return True + return not baseline.get("metrics") + + +def print_paste_me(timings: dict) -> None: + """Print captured timings in the exact baseline-timings.json format. + + The output between the two banner lines is the file contents to paste, + byte-for-byte — sorted keys, 2-space indent, trailing newline. + """ + banner = "=" * 72 + print(banner, file=sys.stderr) + print( + " NO BASELINE FOUND — paste the JSON below into", + file=sys.stderr, + ) + print( + " docker/telemetry/workload/baselines/baseline-timings.json", + file=sys.stderr, + ) + print(banner, file=sys.stderr) + + print(json.dumps(timings, indent=2, sort_keys=True)) + + print(banner, file=sys.stderr) + print( + " (End of paste-me JSON. Gate did NOT run — baseline is empty.)", + file=sys.stderr, + ) + print(banner, file=sys.stderr) + + +def resolve_thresholds( + key: str, + thresholds: dict, +) -> tuple[float | None, float | None]: + """Return ``(pct_threshold, abs_threshold)`` for a metric key. + + Per-metric overrides win over defaults. Returns ``(None, None)`` if no + threshold is defined for this category/quantile — such metrics are + captured but never gate the build. + """ + parts = key.split(".") + if len(parts) < 3: + return (None, None) + category_key = parts[0] + quantile_key = parts[-1] + + category_map = { + "span": "span", + "rpc": "rpc_method", + "job": "job_queue", + } + cat = category_map.get(category_key) + if cat is None: + return (None, None) + + override_key = f"{category_key}.{'.'.join(parts[1:-1])}" + overrides = thresholds.get("overrides", {}) + defaults = thresholds.get("defaults", {}).get(cat, {}) + + rule = overrides.get(override_key, {}).get(quantile_key) + if rule is None: + rule = defaults.get(quantile_key) + if rule is None: + return (None, None) + + pct = rule.get("max_pct_increase") + abs_bound = rule.get("max_abs_increase_ms") + if abs_bound is None: + abs_bound = rule.get("max_abs_increase_us") + return (pct, abs_bound) + + +def _skip_delta( + key: str, + baseline: float | None, + current: float | None, + unit: str, + thresholds: dict, + note: str, +) -> MetricDelta: + """Build a MetricDelta for cases where comparison is not possible.""" + pct_threshold, abs_threshold = resolve_thresholds(key, thresholds) + return MetricDelta( + key=key, + baseline=baseline, + current=current, + delta=None, + pct_change=None, + unit=unit, + threshold_pct=pct_threshold, + threshold_abs=abs_threshold, + regressed=False, + note=note, + ) + + +def compute_delta( + key: str, + baseline_entry: dict | None, + current_entry: dict | None, + thresholds: dict, +) -> MetricDelta: + """Compute a MetricDelta for one metric key. + + A regression requires BOTH bounds to be breached simultaneously. This + tolerates small-value noise: a 100% increase on a 0.5 ms metric + (to 1.0 ms) is not a regression under a 5 ms absolute bound. + """ + baseline = baseline_entry.get("value") if baseline_entry else None + current = current_entry.get("value") if current_entry else None + unit = (baseline_entry or current_entry or {}).get("unit", "") + + if baseline is None and current is None: + return _skip_delta( + key, None, None, unit, thresholds, "no data (neither baseline nor current)" + ) + + if baseline is None: + return _skip_delta( + key, None, current, unit, thresholds, "new metric (not in baseline)" + ) + + if current is None: + return _skip_delta( + key, baseline, None, unit, thresholds, "not captured in current run" + ) + + pct_threshold, abs_threshold = resolve_thresholds(key, thresholds) + delta = current - baseline + pct_change = (delta / baseline * 100.0) if baseline > 0 else None + + if pct_threshold is None or abs_threshold is None: + return MetricDelta( + key=key, + baseline=baseline, + current=current, + delta=delta, + pct_change=pct_change, + unit=unit, + threshold_pct=pct_threshold, + threshold_abs=abs_threshold, + regressed=False, + note="no threshold configured", + ) + + pct_breach = pct_change is not None and pct_change > pct_threshold + abs_breach = delta > abs_threshold + regressed = pct_breach and abs_breach + + if regressed: + note = "REGRESSION" + elif delta < 0: + note = "improved" + else: + note = "within bounds" + + return MetricDelta( + key=key, + baseline=baseline, + current=current, + delta=delta, + pct_change=pct_change, + unit=unit, + threshold_pct=pct_threshold, + threshold_abs=abs_threshold, + regressed=regressed, + note=note, + ) + + +def print_summary(deltas: list[MetricDelta]) -> None: + """Print a sorted, human-readable table of per-metric results.""" + regressions = [d for d in deltas if d.regressed] + improvements = [ + d + for d in deltas + if d.delta is not None and d.delta < 0 and d.baseline not in (None, 0) + ] + improvements.sort(key=lambda d: d.pct_change or 0) + regressions.sort(key=lambda d: -(d.pct_change or 0)) + + print("=" * 72) + print(f" Regression check: {len(regressions)} regression(s) detected") + print("=" * 72) + + if regressions: + print("\nRegressions (breached BOTH pct AND absolute bounds):") + _print_table(regressions) + + if improvements: + top = improvements[:5] + print("\nTop improvements:") + _print_table(top) + + missing = [d for d in deltas if d.note == "not captured in current run"] + if missing: + print(f"\n{len(missing)} baseline metric(s) not captured in current run:") + for d in missing: + print(f" {d.key}") + + +def _print_table(rows: list[MetricDelta]) -> None: + """Print a fixed-width table for a list of deltas.""" + header = f" {'METRIC':<45} {'BASE':>10} {'CUR':>10} {'Δ':>10} {'%':>8} UNIT" + print(header) + print(" " + "-" * (len(header) - 2)) + for d in rows: + base = f"{d.baseline:.2f}" if d.baseline is not None else "-" + cur = f"{d.current:.2f}" if d.current is not None else "-" + delta = f"{d.delta:+.2f}" if d.delta is not None else "-" + pct = f"{d.pct_change:+.1f}%" if d.pct_change is not None else "-" + print(f" {d.key:<45} {base:>10} {cur:>10} {delta:>10} {pct:>8} {d.unit}") + + +def write_report( + deltas: list[MetricDelta], + report_path: Path, + baseline: dict, + timings: dict, +) -> None: + """Write regression-report.json — machine-readable artifact for CI.""" + regressions = [d for d in deltas if d.regressed] + payload = { + "schema_version": 1, + "baseline_captured_at": baseline.get("captured_at"), + "baseline_git_sha": baseline.get("git_sha"), + "current_captured_at": timings.get("captured_at"), + "current_git_sha": timings.get("git_sha"), + "window": timings.get("window"), + "profile": timings.get("profile"), + "summary": { + "total": len(deltas), + "regressions": len(regressions), + "improvements": sum( + 1 + for d in deltas + if d.delta is not None and d.delta < 0 and d.baseline not in (None, 0) + ), + "missing_in_current": sum( + 1 for d in deltas if d.note == "not captured in current run" + ), + }, + "metrics": [asdict(d) for d in deltas], + } + report_path.parent.mkdir(parents=True, exist_ok=True) + with open(report_path, "w") as f: + json.dump(payload, f, indent=2, sort_keys=True) + f.write("\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--timings", + type=Path, + required=True, + help="Captured timings JSON (from capture_timings.py)", + ) + parser.add_argument( + "--baseline", + type=Path, + required=True, + help="Committed baseline-timings.json", + ) + parser.add_argument( + "--thresholds", + type=Path, + default=Path(__file__).parent / "regression-thresholds.json", + help="Threshold policy JSON", + ) + parser.add_argument( + "--report", + type=Path, + default=None, + help="Where to write regression-report.json (optional)", + ) + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(levelname)s %(name)s: %(message)s", + ) + + try: + timings = load_json(args.timings) + baseline = load_json(args.baseline) + thresholds = load_json(args.thresholds) + except (OSError, json.JSONDecodeError) as exc: + logger.error("failed to load inputs: %s", exc) + return 2 + + if is_placeholder(baseline): + print_paste_me(timings) + return 0 + + baseline_metrics = baseline.get("metrics", {}) + current_metrics = timings.get("metrics", {}) + + all_keys = sorted(set(baseline_metrics) | set(current_metrics)) + deltas = [ + compute_delta( + key, + baseline_metrics.get(key), + current_metrics.get(key), + thresholds, + ) + for key in all_keys + ] + + print_summary(deltas) + + if args.report: + write_report(deltas, args.report, baseline, timings) + logger.info("wrote %s", args.report) + + return 1 if any(d.regressed for d in deltas) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/telemetry/workload/expected_metrics.json b/docker/telemetry/workload/expected_metrics.json new file mode 100644 index 00000000000..0c1926e878b --- /dev/null +++ b/docker/telemetry/workload/expected_metrics.json @@ -0,0 +1,149 @@ +{ + "description": "Expected metric inventory for xrpld telemetry validation. Metric names have no prefix (the xrpld_ prefix was removed). beast::insight metrics are lowercased by formatName. Sourced from the live Grafana dashboards and MetricsRegistry.cpp.", + "spanmetrics": { + "description": "SpanMetrics-derived RED metrics from the OTel Collector spanmetrics connector.", + "metrics": [ + "span_calls_total", + "span_duration_milliseconds_bucket", + "span_duration_milliseconds_count", + "span_duration_milliseconds_sum" + ], + "required_labels": [ + "span_name", + "status_code", + "service_name", + "span_kind" + ], + "dimension_labels": [ + "command", + "rpc_status", + "consensus_mode", + "local", + "proposal_trusted", + "validation_trusted", + "tx_type", + "ter_result", + "stage", + "txq_status", + "close_time_correct", + "consensus_state", + "suppressed" + ], + "_dimension_labels_note": "Bare label names as configured in otel-collector-config.yaml spanmetrics dimensions. Informational only (not asserted by the validator)." + }, + "statsd_gauges": { + "description": "beast::insight gauges exported via OTLP/HTTP to the collector (server=otel).", + "metrics": [ + "ledgermaster_validated_ledger_age", + "ledgermaster_published_ledger_age", + "state_accounting_full_duration", + "peer_finder_active_inbound_peers", + "peer_finder_active_outbound_peers", + "jobq_job_count" + ] + }, + "statsd_counters": { + "description": "beast::insight counters exported via OTLP/HTTP. The OTel Prometheus exporter appends _total to monotonic counters.", + "metrics": ["rpc_requests_total", "ledger_fetches_total"] + }, + "overlay_traffic": { + "description": "Overlay traffic metrics (subset — full list has 45+ categories).", + "metrics": [ + "total_bytes_in", + "total_bytes_out", + "total_messages_in", + "total_messages_out" + ] + }, + "phase9_nodestore": { + "description": "Phase 9 NodeStore I/O observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label distinguishing sub-metrics.", + "metrics": ["nodestore_state"] + }, + "phase9_cache": { + "description": "Phase 9 cache hit rate observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.", + "metrics": ["cache_metrics"] + }, + "phase9_txq": { + "description": "Phase 9 transaction queue observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.", + "metrics": ["txq_metrics"] + }, + "phase9_rpc_method": { + "description": "Phase 9 per-RPC-method counters (MetricsRegistry via OTLP).", + "metrics": ["rpc_method_started_total"] + }, + "rpc_in_flight": { + "description": "In-flight RPC gauge via the XRPL_METRIC_UPDOWN_ADD call-site macro (PerfLogImp.cpp, +1 rpcStart / -1 rpcEnd). UpDownCounter: no _total suffix.", + "metrics": ["rpc_in_flight_requests"] + }, + "phase9_objects": { + "description": "Phase 9 counted object instances observable gauge (MetricsRegistry via OTLP).", + "metrics": ["object_count"] + }, + "phase9_load": { + "description": "Phase 9 fee escalation and load factor observable gauge (MetricsRegistry via OTLP).", + "metrics": ["load_factor_metrics"] + }, + "parity_validation_agreement": { + "description": "External dashboard parity: validation agreement percentages (MetricsRegistry).", + "metrics": [ + "validation_agreement{metric=\"agreement_pct_1h\"}", + "validation_agreement{metric=\"agreement_pct_24h\"}" + ] + }, + "parity_validator_health": { + "description": "External dashboard parity: validator health indicators (MetricsRegistry).", + "metrics": [ + "validator_health{metric=\"amendment_blocked\"}", + "validator_health{metric=\"unl_expiry_days\"}" + ] + }, + "parity_peer_quality": { + "description": "External dashboard parity: peer quality metrics (MetricsRegistry).", + "metrics": [ + "peer_quality{metric=\"peer_latency_p90_ms\"}", + "peer_quality{metric=\"peers_insane_count\"}" + ] + }, + "parity_ledger_economy": { + "description": "External dashboard parity: ledger economy metrics (MetricsRegistry).", + "metrics": [ + "ledger_economy{metric=\"base_fee_xrp\"}", + "ledger_economy{metric=\"transaction_rate\"}" + ] + }, + "parity_state_tracking": { + "description": "External dashboard parity: server state tracking (MetricsRegistry).", + "metrics": ["state_tracking{metric=\"state_value\"}"] + }, + "parity_counters": { + "description": "External dashboard parity: monotonic counters (MetricsRegistry).", + "metrics": [ + "ledgers_closed_total", + "validations_sent_total", + "state_changes_total" + ] + }, + "parity_storage": { + "description": "External dashboard parity: storage detail metrics (MetricsRegistry).", + "metrics": ["storage_detail{metric=\"nudb_bytes\"}"] + }, + "grafana_dashboards": { + "description": "All Grafana dashboards that must render data (UIDs as provisioned on disk under docker/telemetry/grafana/dashboards/).", + "uids": [ + "rpc-performance", + "transaction-overview", + "consensus-health", + "ledger-operations", + "peer-network", + "peer-quality", + "fee-market", + "job-queue", + "validator-health", + "node-health", + "network-traffic", + "rpc-pathfinding", + "overlay-traffic-detail", + "ledger-data-sync" + ] + } +} diff --git a/docker/telemetry/workload/expected_spans.json b/docker/telemetry/workload/expected_spans.json new file mode 100644 index 00000000000..f663f303cf3 --- /dev/null +++ b/docker/telemetry/workload/expected_spans.json @@ -0,0 +1,419 @@ +{ + "description": "Expected span inventory for xrpld telemetry validation. Attribute keys follow the 2026-05-13 span-attr naming redesign (bare/underscore form; dotted xrpl.* reserved for resource attributes). Sourced from the *SpanNames.h headers. Spans marked \"optional\": true are conditional — they only fire under traffic the harness may not produce (e.g. gRPC client, missing-ledger fetch, mode transitions) and are not failed when absent.", + "spans": [ + { + "name": "rpc.ws_message", + "category": "rpc", + "parent": null, + "required_attributes": ["command"], + "config_flag": "trace_rpc", + "note": "WebSocket RPC root span. The load generator uses WS, so this is the RPC entry span (not rpc.http_request, which needs an HTTP/JSON-RPC client)." + }, + { + "name": "rpc.process", + "category": "rpc", + "parent": "rpc.ws_message", + "required_attributes": [], + "config_flag": "trace_rpc" + }, + { + "name": "rpc.command.*", + "category": "rpc", + "parent": "rpc.process", + "required_attributes": ["command", "version", "rpc_role", "rpc_status"], + "config_flag": "trace_rpc", + "note": "Wildcard — matches rpc.command.server_info, rpc.command.ledger, etc." + }, + { + "name": "rpc.http_request", + "category": "rpc", + "parent": null, + "required_attributes": ["request_payload_size"], + "config_flag": "trace_rpc", + "optional": true, + "note": "HTTP/JSON-RPC root span. The harness load generator is WebSocket-only, so this does not fire." + }, + { + "name": "tx.process", + "category": "transaction", + "parent": null, + "required_attributes": ["tx_hash", "local", "path"], + "config_flag": "trace_transactions" + }, + { + "name": "tx.receive", + "category": "transaction", + "parent": null, + "required_attributes": ["tx_hash", "peer_id", "suppressed"], + "config_flag": "trace_transactions", + "note": "Cross-node span: parent context propagated from the sender's tx.process via protobuf. Also carries tx_type and peer_version. tx_status is only set when a tx is suppressed/known-bad, so it is not a required attribute on every tx.receive." + }, + { + "name": "tx.apply", + "category": "transaction", + "parent": "ledger.build", + "required_attributes": ["tx_count", "tx_failed"], + "config_flag": "trace_transactions", + "note": "Apply-step span inside BuildLedger. Carries tx_count/tx_failed (ledger_seq lives on the parent ledger.build span)." + }, + { + "name": "tx.preflight", + "category": "transaction", + "parent": null, + "required_attributes": ["stage", "tx_type", "ter_result"], + "config_flag": "trace_transactions", + "note": "Apply-pipeline stage span (stage=preflight). Shares a deterministic trace_id (txID[0:16]) with tx.preclaim/tx.transactor." + }, + { + "name": "tx.preclaim", + "category": "transaction", + "parent": null, + "required_attributes": ["stage", "tx_type", "ter_result"], + "config_flag": "trace_transactions", + "note": "Apply-pipeline stage span (stage=preclaim)." + }, + { + "name": "tx.transactor", + "category": "transaction", + "parent": null, + "required_attributes": ["stage", "tx_type"], + "config_flag": "trace_transactions", + "note": "Apply-pipeline stage span (stage=apply). Also carries applied." + }, + { + "name": "txq.enqueue", + "category": "transaction", + "parent": "tx.process", + "required_attributes": ["tx_hash", "tx_type", "txq_status"], + "config_flag": "trace_transactions", + "optional": true, + "note": "Only fires when a tx is queued (fee below open-ledger level). Requires fee escalation — driven by the txq-burst workload phase. tx_hash/tx_type/txq_status are set on every code path; fee_level_paid/required_fee_level are conditional (TxQ.cpp ~895-898, after the rejected and applied_direct early exits), so they are NOT guaranteed on every txq.enqueue span and cannot be required." + }, + { + "name": "txq.apply_direct", + "category": "transaction", + "parent": "txq.enqueue", + "required_attributes": [], + "config_flag": "trace_transactions", + "optional": true, + "note": "Child of txq.enqueue when the tx applies directly without queueing." + }, + { + "name": "txq.batch_clear", + "category": "transaction", + "parent": "txq.enqueue", + "required_attributes": ["num_cleared"], + "config_flag": "trace_transactions", + "optional": true + }, + { + "name": "txq.accept", + "category": "transaction", + "parent": null, + "required_attributes": ["queue_size", "ledger_changed"], + "config_flag": "trace_transactions", + "optional": true, + "note": "Ledger-close accept loop. Fires on the consensus thread; only meaningful when the queue is non-empty." + }, + { + "name": "txq.accept_tx", + "category": "transaction", + "parent": "txq.accept", + "required_attributes": [ + "tx_hash", + "ter_code", + "retries_remaining", + "txq_status" + ], + "config_flag": "trace_transactions", + "optional": true + }, + { + "name": "txq.cleanup", + "category": "transaction", + "parent": null, + "required_attributes": ["ledger_seq", "expired_count"], + "config_flag": "trace_transactions", + "optional": true + }, + { + "name": "consensus.round", + "category": "consensus", + "parent": null, + "required_attributes": [ + "consensus_ledger_id", + "ledger_seq", + "consensus_mode", + "consensus_round_id", + "consensus_phase" + ], + "config_flag": "trace_consensus", + "note": "Root consensus span created per round. Also carries trace_strategy, previous_ledger_seq, previous_proposers, previous_round_time_ms." + }, + { + "name": "consensus.phase.open", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": [], + "config_flag": "trace_consensus" + }, + { + "name": "consensus.proposal.send", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": ["consensus_round"], + "config_flag": "trace_consensus", + "note": "Also carries is_bow_out." + }, + { + "name": "consensus.ledger_close", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": ["ledger_seq", "consensus_mode"], + "config_flag": "trace_consensus", + "note": "Also carries tx_count_open, close_time_resolution_ms." + }, + { + "name": "consensus.establish", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": [ + "converge_percent", + "establish_count", + "proposers", + "disputes_count" + ], + "config_flag": "trace_consensus" + }, + { + "name": "consensus.update_positions", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": [ + "converge_percent", + "proposers", + "disputes_count" + ], + "config_flag": "trace_consensus" + }, + { + "name": "consensus.check", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": [ + "agree_count", + "disagree_count", + "threshold_percent", + "consensus_result" + ], + "config_flag": "trace_consensus" + }, + { + "name": "consensus.accept", + "category": "consensus", + "parent": "consensus.round", + "required_attributes": ["proposers", "round_time_ms", "quorum"], + "config_flag": "trace_consensus" + }, + { + "name": "consensus.accept.apply", + "category": "consensus", + "parent": "consensus.accept", + "required_attributes": [ + "ledger_seq", + "close_time", + "parent_close_time", + "close_time_self", + "close_time_vote_bins", + "resolution_direction" + ], + "config_flag": "trace_consensus", + "note": "Also carries close_time_correct, close_resolution_ms, consensus_state, proposing, round_time_ms, tx_count." + }, + { + "name": "consensus.validation.send", + "category": "consensus", + "parent": null, + "required_attributes": [ + "ledger_seq", + "proposing", + "ledger_hash", + "full_validation" + ], + "config_flag": "trace_consensus", + "note": "follows-from consensus.accept. ledger_hash and full_validation are shared with peer.validation.receive (same keys, told apart by span name). Also carries validation_sign_time." + }, + { + "name": "consensus.proposal.receive", + "category": "consensus", + "parent": null, + "required_attributes": [], + "config_flag": "trace_consensus", + "note": "Context-propagated from the sending peer. No required local attributes." + }, + { + "name": "consensus.validation.receive", + "category": "consensus", + "parent": null, + "required_attributes": [], + "config_flag": "trace_consensus", + "note": "Context-propagated from the sending peer. No required local attributes." + }, + { + "name": "consensus.mode_change", + "category": "consensus", + "parent": null, + "required_attributes": ["mode_old", "mode_new"], + "config_flag": "trace_consensus", + "optional": true, + "note": "Only fires on an operating-mode transition; a steady cluster rarely changes mode after warmup." + }, + { + "name": "ledger.build", + "category": "ledger", + "parent": null, + "required_attributes": [ + "ledger_seq", + "close_time", + "close_time_correct", + "close_resolution_ms" + ], + "config_flag": "trace_ledger", + "note": "tx_count/tx_failed live on the child tx.apply span, not here." + }, + { + "name": "ledger.validate", + "category": "ledger", + "parent": null, + "required_attributes": ["ledger_seq", "validations"], + "config_flag": "trace_ledger" + }, + { + "name": "ledger.store", + "category": "ledger", + "parent": null, + "required_attributes": ["ledger_seq"], + "config_flag": "trace_ledger" + }, + { + "name": "ledger.acquire", + "category": "ledger", + "parent": null, + "required_attributes": [ + "ledger_seq", + "acquire_reason", + "timeouts", + "peer_count", + "outcome" + ], + "config_flag": "trace_ledger", + "optional": true, + "note": "Only fires when a node must fetch a missing ledger (InboundLedger). A healthy local cluster rarely back-fills history." + }, + { + "name": "peer.proposal.receive", + "category": "peer", + "parent": null, + "required_attributes": ["peer_id", "proposal_trusted"], + "config_flag": "trace_peer" + }, + { + "name": "peer.validation.receive", + "category": "peer", + "parent": null, + "required_attributes": [ + "peer_id", + "validation_trusted", + "ledger_hash", + "full_validation" + ], + "config_flag": "trace_peer", + "note": "ledger_hash and full_validation are shared with consensus.validation.send (same keys, told apart by span name)." + }, + { + "name": "pathfind.request", + "category": "pathfind", + "parent": null, + "required_attributes": [ + "pathfind_source_account", + "pathfind_dest_account" + ], + "config_flag": "trace_rpc", + "note": "Fires on ripple_path_find / path_find RPC. Driven by the ripple_path_find load in rpc_load_generator.py." + }, + { + "name": "pathfind.compute", + "category": "pathfind", + "parent": "pathfind.request", + "required_attributes": ["pathfind_fast"], + "config_flag": "trace_rpc", + "optional": true, + "note": "Only fires when PathRequest::doUpdate runs a computation; the self-to-self XRP probe from the load generator returns early without computing paths in a fresh cluster with no liquidity." + }, + { + "name": "pathfind.discover", + "category": "pathfind", + "parent": "pathfind.compute", + "required_attributes": ["pathfind_search_level", "pathfind_num_paths"], + "config_flag": "trace_rpc", + "optional": true, + "note": "Graph exploration; only fires under pathfind.compute, which needs real path liquidity not present in the fresh test cluster." + }, + { + "name": "pathfind.update_all", + "category": "pathfind", + "parent": null, + "required_attributes": ["pathfind_ledger_index", "pathfind_num_requests"], + "config_flag": "trace_rpc", + "optional": true, + "note": "Async recomputation at ledger close; only fires when there are active path_find subscriptions (the one-shot ripple_path_find load does not register one)." + }, + { + "name": "grpc.*", + "category": "grpc", + "parent": null, + "required_attributes": ["method", "grpc_role", "grpc_status"], + "config_flag": "trace_rpc", + "optional": true, + "note": "Wildcard — grpc.. The harness has no gRPC client, so these do not fire. Tracked for completeness." + } + ], + "parent_child_relationships": [ + { + "parent": "rpc.ws_message", + "child": "rpc.process", + "description": "WebSocket message contains processing span", + "skip": true, + "skip_reason": "rpc.ws_message and rpc.process run on different threads (the WS handler posts a coroutine to JobQueue for processing). Span context is not propagated across the thread boundary. Requires a C++ fix to capture and forward the span context through the coroutine lambda." + }, + { + "parent": "rpc.process", + "child": "rpc.command.*", + "description": "Processing span contains per-command span" + }, + { + "parent": "ledger.build", + "child": "tx.apply", + "description": "Ledger build contains transaction application" + }, + { + "parent": "consensus.round", + "child": "consensus.accept", + "description": "Consensus round contains the accept sub-span" + }, + { + "parent": "consensus.accept", + "child": "consensus.accept.apply", + "description": "Accept contains the ledger-apply sub-span" + }, + { + "parent": "pathfind.request", + "child": "pathfind.compute", + "description": "Pathfind request contains the compute sub-span", + "skip": true, + "skip_reason": "pathfind.compute only fires when a path computation actually runs; the self-to-self XRP probe in a fresh cluster with no liquidity returns before computing, so the child is not emitted under the harness workload." + } + ], + "total_span_types": 40, + "total_unique_attributes": 58 +} diff --git a/docker/telemetry/workload/generate-validator-keys.sh b/docker/telemetry/workload/generate-validator-keys.sh new file mode 100755 index 00000000000..7324a8bf61e --- /dev/null +++ b/docker/telemetry/workload/generate-validator-keys.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# generate-validator-keys.sh — Generate validator key pairs for the workload harness. +# +# Uses a temporary standalone xrpld instance to call `validation_create` RPC +# for each node. Outputs a JSON file mapping node index to seed + public key. +# +# Usage: +# ./generate-validator-keys.sh +# +# Output: +# /validator-keys.json — JSON array of {index, seed, public_key} +# /validators.txt — [validators] section for xrpld.cfg + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colored output helpers +# --------------------------------------------------------------------------- +log() { printf "\033[1;34m[KEYGEN]\033[0m %s\n" "$*"; } +ok() { printf "\033[1;32m[KEYGEN]\033[0m %s\n" "$*"; } +die() { + printf "\033[1;31m[KEYGEN]\033[0m %s\n" "$*" >&2 + exit 1 +} + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +usage() { + echo "Usage: $0 " + echo "" + echo "Arguments:" + echo " xrpld_binary Path to xrpld binary (built with telemetry=ON)" + echo " num_nodes Number of validator key pairs to generate (1-20)" + echo " output_dir Directory to write validator-keys.json and validators.txt" + exit 1 +} + +if [ $# -lt 3 ]; then + usage +fi + +XRPLD="$1" +NUM_NODES="$2" +OUTPUT_DIR="$3" + +# Validate arguments +[ -x "$XRPLD" ] || die "xrpld binary not found or not executable: $XRPLD" +[[ "$NUM_NODES" =~ ^[0-9]+$ ]] || die "num_nodes must be a positive integer" +[ "$NUM_NODES" -ge 1 ] && [ "$NUM_NODES" -le 20 ] || die "num_nodes must be between 1 and 20" + +mkdir -p "$OUTPUT_DIR" + +# --------------------------------------------------------------------------- +# Start a temporary standalone xrpld for key generation +# --------------------------------------------------------------------------- +TEMP_DIR="$(mktemp -d)" +TEMP_PORT=5099 +TEMP_CFG="$TEMP_DIR/xrpld.cfg" + +log "Starting temporary xrpld for key generation (port $TEMP_PORT)..." + +cat >"$TEMP_CFG" <"$TEMP_DIR/stdout.log" 2>&1 & +TEMP_PID=$! + +# Ensure cleanup on exit +cleanup_temp() { + kill "$TEMP_PID" 2>/dev/null || true + wait "$TEMP_PID" 2>/dev/null || true + rm -rf "$TEMP_DIR" +} +trap cleanup_temp EXIT + +# Wait for RPC to become available +for attempt in $(seq 1 30); do + if curl -sf "http://localhost:$TEMP_PORT" \ + -d '{"method":"server_info"}' >/dev/null 2>&1; then + log "Temporary xrpld RPC ready (attempt $attempt)." + break + fi + if [ "$attempt" -eq 30 ]; then + die "Temporary xrpld RPC not ready after 30s" + fi + sleep 1 +done + +# --------------------------------------------------------------------------- +# Generate key pairs +# --------------------------------------------------------------------------- +log "Generating $NUM_NODES validator key pairs..." + +KEYS_JSON="[" +VALIDATORS_TXT="[validators]" + +for i in $(seq 1 "$NUM_NODES"); do + result=$(curl -sf "http://localhost:$TEMP_PORT" \ + -d '{"method":"validation_create"}') + seed=$(echo "$result" | jq -r '.result.validation_seed') + pubkey=$(echo "$result" | jq -r '.result.validation_public_key') + + if [ -z "$seed" ] || [ "$seed" = "null" ]; then + die "Failed to generate key pair for node $i" + fi + + log " Node $i: ${pubkey:0:20}..." + + # Build JSON entry + entry="{\"index\": $i, \"seed\": \"$seed\", \"public_key\": \"$pubkey\"}" + if [ "$i" -gt 1 ]; then + KEYS_JSON="$KEYS_JSON," + fi + KEYS_JSON="$KEYS_JSON$entry" + + VALIDATORS_TXT="$VALIDATORS_TXT +$pubkey" +done + +KEYS_JSON="$KEYS_JSON]" + +# --------------------------------------------------------------------------- +# Write output files +# --------------------------------------------------------------------------- +echo "$KEYS_JSON" | jq '.' >"$OUTPUT_DIR/validator-keys.json" +echo "$VALIDATORS_TXT" >"$OUTPUT_DIR/validators.txt" + +ok "Generated $NUM_NODES key pairs:" +ok " Keys: $OUTPUT_DIR/validator-keys.json" +ok " Validators: $OUTPUT_DIR/validators.txt" diff --git a/docker/telemetry/workload/prom_queries.py b/docker/telemetry/workload/prom_queries.py new file mode 100644 index 00000000000..b257c61241e --- /dev/null +++ b/docker/telemetry/workload/prom_queries.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Shared Prometheus query helpers for the regression gate. + +Single source of truth for how regression metrics are computed. Both +``capture_timings.py`` and any future tooling consume this module so metric +name → PromQL expression stays consistent. + +Design: +- Every captured metric has a key in the form ``{category}.{name}.p{quantile}`` + (e.g. ``span.tx.process.p99``). Keys are flat strings so JSON diffing is + trivial. +- Quantile queries go through ``histogram_quantile`` over the standard + ``_bucket`` series. The rate window is a parameter (defaults to the + capture window, not Prometheus's default 5m) so short CI runs are usable. +- The catalogue of what to capture lives in ``regression-metrics.json`` — + this module only knows how to translate that JSON into HTTP queries. + +Usage:: + + import asyncio, aiohttp + from prom_queries import build_query_plan, run_query_plan + + plan = build_query_plan("regression-metrics.json", window="3m") + async with aiohttp.ClientSession() as s: + timings = await run_query_plan(s, "http://localhost:9090", plan) + # timings = {"span.tx.process.p99": {"value": 12.4, "unit": "ms"}, ...} +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import aiohttp + +logger = logging.getLogger("prom_queries") + + +@dataclass(frozen=True) +class QueryEntry: + """One metric to capture from Prometheus. + + Attributes: + key: Flat output key, e.g. ``span.tx.process.p99``. + promql: The PromQL expression to send to /api/v1/query. + unit: Unit of the returned value, e.g. ``ms`` or ``us``. + Baseline JSON preserves this so the comparator can + sanity-check unit drift. + """ + + key: str + promql: str + unit: str + + +def _build_simple_entries( + cfg: dict, + prefix: str, + window: str, +) -> list[QueryEntry]: + """Build QueryEntry list for a single-template category (spans, rpc).""" + tmpl = cfg.get("_query_template", "") + unit = cfg.get("_unit", "ms") + entries: list[QueryEntry] = [] + for name in cfg.get("names", []): + for q in cfg.get("_quantiles", []): + expr = ( + tmpl.replace("{quantile}", _format_quantile(q)) + .replace("{name}", name) + .replace("{window}", window) + ) + entries.append( + QueryEntry( + key=f"{prefix}.{name}.p{_quantile_label(q)}", + promql=expr, + unit=unit, + ) + ) + return entries + + +def _build_job_entries(cfg: dict, window: str) -> list[QueryEntry]: + """Build QueryEntry list for the job_queue category (multi-phase).""" + unit = cfg.get("_unit", "us") + phases = cfg.get("_phases", ["queued", "running"]) + tmpl_map = { + "queued": cfg.get("_queued_template", ""), + "running": cfg.get("_running_template", ""), + } + entries: list[QueryEntry] = [] + for name in cfg.get("names", []): + for phase in phases: + tmpl = tmpl_map.get(phase, "") + if not tmpl: + continue + for q in cfg.get("_quantiles", []): + expr = ( + tmpl.replace("{quantile}", _format_quantile(q)) + .replace("{name}", name) + .replace("{window}", window) + ) + entries.append( + QueryEntry( + key=f"job.{name}.{phase}.p{_quantile_label(q)}", + promql=expr, + unit=unit, + ) + ) + return entries + + +def build_query_plan(metrics_path: str | Path, window: str = "3m") -> list[QueryEntry]: + """Translate regression-metrics.json into a list of PromQL queries. + + Args: + metrics_path: Path to ``regression-metrics.json``. + window: Rate window passed to ``rate()``. For short CI runs + keep this close to the test duration so the bucket + counts are meaningful. Default 3m matches the + ``regression`` workload profile. + + Returns: + A list of ``QueryEntry`` values, one per (metric × quantile). + """ + with open(metrics_path) as f: + cfg = json.load(f) + + plan: list[QueryEntry] = [] + plan.extend(_build_simple_entries(cfg.get("spans", {}), "span", window)) + plan.extend(_build_simple_entries(cfg.get("rpc_methods", {}), "rpc", window)) + plan.extend(_build_job_entries(cfg.get("job_queue", {}), window)) + return plan + + +async def run_query_plan( + session: aiohttp.ClientSession, + prom_url: str, + plan: list[QueryEntry], +) -> dict[str, dict[str, Any]]: + """Execute a query plan and return a flat ``key → {value, unit}`` map. + + Queries that return no data (NaN, empty result) are still included in + the output with ``value: null`` — the comparator treats missing values + as "not yet observed" rather than as a regression. This keeps the + baseline schema stable across runs with different load levels. + + Args: + session: Shared aiohttp session. + prom_url: Base URL of Prometheus (e.g. ``http://localhost:9090``). + plan: Output of :func:`build_query_plan`. + + Returns: + Mapping from metric key to ``{"value": float|None, "unit": str}``. + """ + results: dict[str, dict[str, Any]] = {} + for entry in plan: + value = await _instant_query(session, prom_url, entry.promql) + results[entry.key] = {"value": value, "unit": entry.unit} + return results + + +async def _instant_query( + session: aiohttp.ClientSession, + prom_url: str, + promql: str, +) -> float | None: + """POST an instant query to Prometheus; return the scalar value or None. + + None is returned for NaN, empty results, or HTTP errors — every call + site treats None identically ("no data captured"). + """ + url = f"{prom_url.rstrip('/')}/api/v1/query" + try: + async with session.post( + url, data={"query": promql}, timeout=aiohttp.ClientTimeout(total=30) + ) as resp: + if resp.status != 200: + logger.warning("query HTTP %d: %s", resp.status, promql) + return None + body = await resp.json() + except (aiohttp.ClientError, TimeoutError) as exc: + logger.warning("query failed: %s — %s", promql, exc) + return None + + if body.get("status") != "success": + logger.warning("query status=%s: %s", body.get("status"), promql) + return None + + result = body.get("data", {}).get("result", []) + if not result: + return None + + raw = result[0].get("value", [None, None])[1] + if raw is None or raw in ("NaN", "+Inf", "-Inf"): + return None + try: + return float(raw) + except (TypeError, ValueError): + return None + + +def _format_quantile(q: float) -> str: + """Format a quantile for PromQL (``0.99`` → ``"0.99"``).""" + return f"{q:g}" + + +def _quantile_label(q: float) -> str: + """Format a quantile for the output key (``0.95`` → ``"95"``).""" + return str(int(round(q * 100))) diff --git a/docker/telemetry/workload/regression-metrics.json b/docker/telemetry/workload/regression-metrics.json new file mode 100644 index 00000000000..f475d86fd2f --- /dev/null +++ b/docker/telemetry/workload/regression-metrics.json @@ -0,0 +1,28 @@ +{ + "_description": "Metric surface for the OTel-driven regression gate. Each entry names a metric, the quantiles to capture, and how to query Prometheus. The comparator compares current run against baseline-timings.json under these exact keys.", + "_key_format": "{category}.{name}.p{quantile} (e.g. span.tx.process.p99, rpc.server_info.p95, job.transaction.queued.p95)", + "spans": { + "_query_template": "histogram_quantile({quantile}, sum by (le) (rate(span_duration_milliseconds_bucket{span_name=\"{name}\"}[{window}])))", + "_unit": "ms", + "_quantiles": [0.5, 0.95, 0.99], + "names": [ + "rpc.ws_message", + "rpc.process", + "tx.process", + "tx.apply", + "ledger.build", + "ledger.validate", + "ledger.store", + "consensus.ledger_close", + "consensus.accept" + ] + }, + "job_queue": { + "_queued_template": "histogram_quantile({quantile}, sum by (le) (rate(job_queued_us_bucket{job_type=\"{name}\"}[{window}])))", + "_running_template": "histogram_quantile({quantile}, sum by (le) (rate(job_running_us_bucket{job_type=\"{name}\"}[{window}])))", + "_unit": "us", + "_quantiles": [0.95], + "_phases": ["queued", "running"], + "names": ["transaction", "acceptLedger"] + } +} diff --git a/docker/telemetry/workload/regression-thresholds.json b/docker/telemetry/workload/regression-thresholds.json new file mode 100644 index 00000000000..ae6789f1ba3 --- /dev/null +++ b/docker/telemetry/workload/regression-thresholds.json @@ -0,0 +1,26 @@ +{ + "_description": "Per-metric regression thresholds. A metric regresses when current - baseline exceeds BOTH the percentage and absolute bounds (AND, not OR — this tolerates small-value noise). Defaults apply unless a per-metric override exists.", + "_bucket_note": "SpanMetrics latency histograms use explicit buckets [1,5,10,25,50,100,250,500,1000,2000,3000,4000,5000,10000,30000]ms. A quantile sitting near a low-end boundary can jump a full bucket (e.g. 1ms->5ms) between runs with no real change, so absolute span bounds are set to ~2 low-end bucket widths (10ms) to tolerate that quantization noise while still catching genuine multi-bucket regressions. Second-scale consensus spans now have 2s/3s/4s boundaries (previously all fell in one 1s-5s bucket); their quantiles quantize to ~1s widths there. The job_queue running bound is widened similarly — per-ledger apply work scales with TxQ burst load.", + "defaults": { + "span": { + "p50": { "max_pct_increase": 50.0, "max_abs_increase_ms": 10.0 }, + "p95": { "max_pct_increase": 50.0, "max_abs_increase_ms": 10.0 }, + "p99": { "max_pct_increase": 50.0, "max_abs_increase_ms": 15.0 } + }, + "job_queue": { + "p95": { "max_pct_increase": 50.0, "max_abs_increase_us": 20000.0 } + } + }, + "overrides": { + "span.consensus.ledger_close": { + "p50": { "max_pct_increase": 5.0, "max_abs_increase_ms": 200.0 }, + "p95": { "max_pct_increase": 5.0, "max_abs_increase_ms": 500.0 }, + "p99": { "max_pct_increase": 5.0, "max_abs_increase_ms": 1000.0 } + }, + "span.consensus.accept": { + "p50": { "max_pct_increase": 5.0, "max_abs_increase_ms": 200.0 }, + "p95": { "max_pct_increase": 5.0, "max_abs_increase_ms": 500.0 }, + "p99": { "max_pct_increase": 5.0, "max_abs_increase_ms": 1000.0 } + } + } +} diff --git a/docker/telemetry/workload/requirements.txt b/docker/telemetry/workload/requirements.txt new file mode 100644 index 00000000000..f115de082b0 --- /dev/null +++ b/docker/telemetry/workload/requirements.txt @@ -0,0 +1,6 @@ +# Python dependencies for Phase 10 workload tools. +# +# Install: pip install -r requirements.txt + +websockets>=12.0 +aiohttp>=3.9.0 diff --git a/docker/telemetry/workload/rpc_load_generator.py b/docker/telemetry/workload/rpc_load_generator.py new file mode 100644 index 00000000000..7b5e22f6318 --- /dev/null +++ b/docker/telemetry/workload/rpc_load_generator.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +"""RPC Load Generator for rippled telemetry validation. + +Connects to one or more rippled WebSocket endpoints and fires all traced +RPC commands at configurable rates with realistic production-like +distribution. + +Command distribution (default weights): + 40% Health checks: server_info, fee + 30% Wallet queries: account_info, account_lines, account_objects + 15% Explorer: ledger, ledger_data + 10% TX lookups: tx, account_tx + 5% DEX queries: book_offers, amm_info + 3% Pathfinding: ripple_path_find + +Usage: + python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120 + + # Multiple endpoints (round-robin): + python3 rpc_load_generator.py \\ + --endpoints ws://localhost:6006 ws://localhost:6007 \\ + --rate 100 --duration 300 + + # Custom weights: + python3 rpc_load_generator.py --endpoints ws://localhost:6006 \\ + --weights '{"server_info":60,"account_info":30,"ledger":10}' +""" + +import argparse +import asyncio +import json +import logging +import random +import sys +import time +import uuid +from dataclasses import dataclass, field +from typing import Any + +import websockets + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +# Default command distribution matching realistic production ratios. +# Keys are RPC command names; values are relative weights. +DEFAULT_WEIGHTS: dict[str, int] = { + # 40% health checks + "server_info": 25, + "fee": 15, + # 30% wallet queries + "account_info": 15, + "account_lines": 8, + "account_objects": 7, + # 15% explorer + "ledger": 10, + "ledger_data": 5, + # 10% tx lookups + "tx": 5, + "account_tx": 5, + # 5% DEX queries + "book_offers": 3, + "amm_info": 2, + # Pathfinding — exercises the pathfind.request/compute/discover spans. + # ripple_path_find is the synchronous (one-shot) variant that fits this + # fire-one-request WS client; path_find is a streaming subscription. + "ripple_path_find": 3, +} + +# Well-known genesis account for queries that require an account parameter. +GENESIS_ACCOUNT = "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh" + +logger = logging.getLogger("rpc_load_generator") + + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclass +class LoadStats: + """Tracks request counts and latencies during a load run. + + Attributes: + total_sent: Total RPC requests dispatched. + total_success: Requests that returned a valid result. + total_errors: Requests that returned an error or timed out. + latencies: Per-command list of round-trip times in seconds. + command_counts: Per-command request count. + """ + + total_sent: int = 0 + total_success: int = 0 + total_errors: int = 0 + latencies: dict[str, list[float]] = field(default_factory=dict) + command_counts: dict[str, int] = field(default_factory=dict) + + def record(self, command: str, latency: float, success: bool) -> None: + """Record the outcome of a single RPC call.""" + self.total_sent += 1 + if success: + self.total_success += 1 + else: + self.total_errors += 1 + self.latencies.setdefault(command, []).append(latency) + self.command_counts[command] = self.command_counts.get(command, 0) + 1 + + def summary(self) -> dict[str, Any]: + """Return a summary dict suitable for JSON serialization.""" + per_command: dict[str, Any] = {} + for cmd, lats in self.latencies.items(): + sorted_lats = sorted(lats) + n = len(sorted_lats) + per_command[cmd] = { + "count": self.command_counts.get(cmd, 0), + "p50_ms": round(sorted_lats[n // 2] * 1000, 2) if n else 0, + "p95_ms": (round(sorted_lats[int(n * 0.95)] * 1000, 2) if n else 0), + "p99_ms": (round(sorted_lats[int(n * 0.99)] * 1000, 2) if n else 0), + } + return { + "total_sent": self.total_sent, + "total_success": self.total_success, + "total_errors": self.total_errors, + "error_rate_pct": ( + round(self.total_errors / self.total_sent * 100, 2) + if self.total_sent + else 0 + ), + "per_command": per_command, + } + + +# --------------------------------------------------------------------------- +# RPC command builders +# --------------------------------------------------------------------------- + + +def build_rpc_request(command: str) -> dict[str, Any]: + """Build a native WebSocket command request for the given command. + + Uses rippled's native WS format (``{"command": ...}``) with flat + parameters, NOT the JSON-RPC format (``{"method": ..., "params": [...]}``). + + Args: + command: The rippled RPC command name. + + Returns: + A dict representing the native WebSocket request body. + """ + req: dict[str, Any] = {"command": command} + + if command in ("server_info", "fee"): + pass # No params needed. + elif command == "account_info": + req["account"] = GENESIS_ACCOUNT + elif command == "account_lines": + req["account"] = GENESIS_ACCOUNT + elif command == "account_objects": + req["account"] = GENESIS_ACCOUNT + req["limit"] = 10 + elif command == "ledger": + req["ledger_index"] = "validated" + elif command == "ledger_data": + req["ledger_index"] = "validated" + req["limit"] = 5 + elif command == "tx": + # Use a dummy hash — returns "txnNotFound" error but still exercises + # the full RPC span pipeline (rpc.ws_message -> rpc.process -> rpc.command.tx). + req["transaction"] = "0" * 64 + req["binary"] = False + elif command == "account_tx": + req["account"] = GENESIS_ACCOUNT + req["ledger_index_min"] = -1 + req["ledger_index_max"] = -1 + req["limit"] = 5 + elif command == "book_offers": + req["taker_pays"] = {"currency": "XRP"} + req["taker_gets"] = { + "currency": "USD", + "issuer": GENESIS_ACCOUNT, + } + req["limit"] = 5 + elif command == "amm_info": + # AMM may not exist — the span is still created on the server side. + req["asset"] = {"currency": "XRP"} + req["asset2"] = { + "currency": "USD", + "issuer": GENESIS_ACCOUNT, + } + elif command == "ripple_path_find": + # Self-to-self XRP path search. It returns no usable paths, but the + # server still runs the full pathfinding pipeline (pathfind.request -> + # pathfind.compute -> pathfind.discover), which is what we trace. + req["source_account"] = GENESIS_ACCOUNT + req["destination_account"] = GENESIS_ACCOUNT + req["destination_amount"] = "1000000" # 1 XRP in drops + + return req + + +def choose_command(weights: dict[str, int]) -> str: + """Select a random RPC command based on configured weights. + + Args: + weights: Mapping of command name to relative weight. + + Returns: + A command name string. + """ + commands = list(weights.keys()) + w = [weights[c] for c in commands] + return random.choices(commands, weights=w, k=1)[0] + + +# --------------------------------------------------------------------------- +# WebSocket RPC client +# --------------------------------------------------------------------------- + + +async def send_rpc( + ws: websockets.WebSocketClientProtocol, + command: str, + stats: LoadStats, + inject_traceparent: bool = True, +) -> None: + """Send a single RPC request over WebSocket and record the result. + + Args: + ws: Open WebSocket connection. + command: RPC command name. + stats: LoadStats instance to record results. + inject_traceparent: If True, add a W3C traceparent header field + to the request for context propagation testing. + """ + request = build_rpc_request(command) + + # Inject W3C traceparent for context propagation testing. + # The rippled WebSocket handler extracts this from the JSON body + # when present (Phase 2 context propagation). + if inject_traceparent: + trace_id = uuid.uuid4().hex + span_id = uuid.uuid4().hex[:16] + request["traceparent"] = f"00-{trace_id}-{span_id}-01" + + t0 = time.monotonic() + try: + await ws.send(json.dumps(request)) + raw = await asyncio.wait_for(ws.recv(), timeout=10.0) + latency = time.monotonic() - t0 + response = json.loads(raw) + # Native WS responses have {"status": "success", "result": {...}} + # or {"status": "error", "error": "...", "error_message": "..."}. + success = response.get("status") == "success" + stats.record(command, latency, success) + except (asyncio.TimeoutError, websockets.exceptions.WebSocketException) as exc: + latency = time.monotonic() - t0 + stats.record(command, latency, False) + logger.debug("RPC %s failed: %s", command, exc) + + +async def run_load( + endpoints: list[str], + rate: float, + duration: float, + weights: dict[str, int], + inject_traceparent: bool, +) -> LoadStats: + """Run the RPC load generator against the given endpoints. + + Distributes requests round-robin across endpoints at the specified + rate (requests per second) for the given duration. + + Args: + endpoints: List of WebSocket URLs (ws://host:port). + rate: Target requests per second. + duration: Total run time in seconds. + weights: Command distribution weights. + inject_traceparent: Whether to inject W3C traceparent headers. + + Returns: + LoadStats with aggregated results. + """ + stats = LoadStats() + interval = 1.0 / rate if rate > 0 else 0.1 + + # Open persistent connections to all endpoints. + connections: list[websockets.WebSocketClientProtocol] = [] + for ep in endpoints: + try: + ws = await websockets.connect(ep, ping_interval=20, ping_timeout=10) + connections.append(ws) + logger.info("Connected to %s", ep) + except Exception as exc: + logger.error("Failed to connect to %s: %s", ep, exc) + + if not connections: + logger.error("No connections established. Aborting.") + return stats + + logger.info( + "Starting load: rate=%s RPS, duration=%ss, endpoints=%d", + rate, + duration, + len(connections), + ) + + start = time.monotonic() + conn_idx = 0 + + try: + while (time.monotonic() - start) < duration: + command = choose_command(weights) + ws = connections[conn_idx % len(connections)] + conn_idx += 1 + + # Fire-and-forget style with bounded concurrency via sleep. + asyncio.create_task(send_rpc(ws, command, stats, inject_traceparent)) + await asyncio.sleep(interval) + + # Periodic progress log. + elapsed = time.monotonic() - start + if stats.total_sent % 100 == 0 and stats.total_sent > 0: + actual_rps = stats.total_sent / elapsed if elapsed > 0 else 0 + logger.info( + "Progress: %d sent, %d errors, %.1f RPS (%.0fs elapsed)", + stats.total_sent, + stats.total_errors, + actual_rps, + elapsed, + ) + except asyncio.CancelledError: + logger.info("Load generation cancelled.") + finally: + # Allow in-flight requests to complete. + await asyncio.sleep(2) + for ws in connections: + await ws.close() + + elapsed = time.monotonic() - start + logger.info( + "Load complete: %d sent, %d success, %d errors in %.1fs (%.1f RPS)", + stats.total_sent, + stats.total_success, + stats.total_errors, + elapsed, + stats.total_sent / elapsed if elapsed > 0 else 0, + ) + + return stats + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="RPC Load Generator for rippled telemetry validation", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Basic usage (50 RPS for 2 minutes): + python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120 + + # Multiple endpoints with custom weights: + python3 rpc_load_generator.py \\ + --endpoints ws://localhost:6006 ws://localhost:6007 \\ + --rate 100 --duration 300 \\ + --weights '{"server_info": 80, "account_info": 20}' + """, + ) + parser.add_argument( + "--endpoints", + nargs="+", + default=["ws://localhost:6006"], + help="WebSocket endpoints (default: ws://localhost:6006)", + ) + parser.add_argument( + "--rate", + type=float, + default=50.0, + help="Target requests per second (default: 50)", + ) + parser.add_argument( + "--duration", + type=float, + default=120.0, + help="Run duration in seconds (default: 120)", + ) + parser.add_argument( + "--weights", + type=str, + default=None, + help="JSON string of command weights (overrides defaults)", + ) + parser.add_argument( + "--no-traceparent", + action="store_true", + help="Disable W3C traceparent injection", + ) + parser.add_argument( + "--output", + type=str, + default=None, + help="Write JSON summary to this file path", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable debug logging", + ) + return parser.parse_args() + + +def main() -> None: + """Main entry point for the RPC load generator.""" + args = parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s %(message)s", + ) + + # Parse custom weights if provided. + weights = DEFAULT_WEIGHTS.copy() + if args.weights: + try: + custom = json.loads(args.weights) + weights = {k: int(v) for k, v in custom.items()} + logger.info("Using custom weights: %s", weights) + except (json.JSONDecodeError, ValueError) as exc: + logger.error("Invalid --weights JSON: %s", exc) + sys.exit(1) + + # Run the load generator. + stats = asyncio.run( + run_load( + endpoints=args.endpoints, + rate=args.rate, + duration=args.duration, + weights=weights, + inject_traceparent=not args.no_traceparent, + ) + ) + + summary = stats.summary() + print(json.dumps(summary, indent=2)) + + if args.output: + with open(args.output, "w") as f: + json.dump(summary, f, indent=2) + logger.info("Summary written to %s", args.output) + + # Exit with error if error rate exceeds 50%. + if summary["error_rate_pct"] > 50: + logger.error("High error rate: %.1f%%", summary["error_rate_pct"]) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/docker/telemetry/workload/run-full-validation.sh b/docker/telemetry/workload/run-full-validation.sh new file mode 100755 index 00000000000..c6c292410b1 --- /dev/null +++ b/docker/telemetry/workload/run-full-validation.sh @@ -0,0 +1,500 @@ +#!/usr/bin/env bash +# run-full-validation.sh — Orchestrates the full telemetry validation pipeline. +# +# Sequence: +# 1. Start the observability stack (OTel Collector, Tempo, Prometheus, Loki, Grafana) +# 2. Start a multi-node rippled cluster with full telemetry enabled +# 3. Wait for consensus +# 4. Run workload orchestrator (RPC load, TX submission, propagation wait) +# 5. Run the telemetry validation suite +# 6. Capture OTel timings and compare against committed baseline +# 7. (Optional) Run the performance overhead benchmark +# +# Usage: +# ./run-full-validation.sh --xrpld /path/to/xrpld +# ./run-full-validation.sh --xrpld /path/to/xrpld --with-benchmark +# ./run-full-validation.sh --xrpld /path/to/xrpld --skip-regression +# ./run-full-validation.sh --cleanup +# +# Exit codes: +# 0 — All validation checks and the regression gate passed +# 1 — Validation checks failed OR the regression gate detected a regression +# 2 — Infrastructure error (cluster/stack failed to start, timing capture failed) + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colored output helpers +# --------------------------------------------------------------------------- +log() { printf "\033[1;34m[VALIDATE]\033[0m %s\n" "$*"; } +ok() { printf "\033[1;32m[VALIDATE]\033[0m %s\n" "$*"; } +warn() { printf "\033[1;33m[VALIDATE]\033[0m %s\n" "$*"; } +fail() { printf "\033[1;31m[VALIDATE]\033[0m %s\n" "$*"; } +die() { + printf "\033[1;31m[VALIDATE]\033[0m %s\n" "$*" >&2 + exit 2 +} + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TELEMETRY_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +REPO_ROOT="$(cd "$TELEMETRY_DIR/../.." && pwd)" +COMPOSE_FILE="$TELEMETRY_DIR/docker-compose.workload.yaml" +WORKDIR="/tmp/xrpld-validation" + +XRPLD="${XRPLD:-$REPO_ROOT/.build/xrpld}" +NUM_NODES=5 +RPC_PORT_BASE=5005 +WS_PORT_BASE=6006 +PEER_PORT_BASE=51235 +RPC_RATE=50 +RPC_DURATION=120 +TX_TPS=5 +TX_DURATION=120 +WITH_BENCHMARK=false +SKIP_LOKI=false +SKIP_REGRESSION=false +WORKLOAD_PROFILE="full-validation" +REPORT_DIR="$WORKDIR/reports" +# Rate window handed to Prometheus `rate()` when capturing timings. Keep +# this close to the active workload duration so histogram buckets cover +# the measurement window; longer windows dilute short-lived regressions. +REGRESSION_WINDOW="${REGRESSION_WINDOW:-3m}" +BASELINE_FILE="${BASELINE_FILE:-$SCRIPT_DIR/baselines/baseline-timings.json}" +THRESHOLDS_FILE="${THRESHOLDS_FILE:-$SCRIPT_DIR/regression-thresholds.json}" +METRICS_FILE="${METRICS_FILE:-$SCRIPT_DIR/regression-metrics.json}" + +GENESIS_ACCOUNT="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh" +GENESIS_SEED="snoPBrXtMeMyMHUVTgbuqAfg1SUTb" + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +usage() { + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --xrpld PATH Path to xrpld binary" + echo " --nodes NUM Number of validator nodes (default: 5)" + echo " --rpc-rate RPS RPC load rate (default: 50)" + echo " --rpc-duration SECS RPC load duration (default: 120)" + echo " --tx-tps TPS Transaction submit rate (default: 5)" + echo " --tx-duration SECS Transaction submit duration (default: 120)" + echo " --profile NAME Workload profile (default: full-validation)" + echo " --with-benchmark Also run performance overhead benchmark (telemetry off vs on)" + echo " --skip-loki Skip Loki log-trace correlation checks" + echo " --skip-regression Skip the OTel-baseline regression gate" + echo " --cleanup Tear down everything and exit" + echo " -h, --help Show this help" + exit 0 +} + +while [ $# -gt 0 ]; do + case "$1" in + --xrpld) + XRPLD="$2" + shift 2 + ;; + --nodes) + NUM_NODES="$2" + shift 2 + ;; + --rpc-rate) + RPC_RATE="$2" + shift 2 + ;; + --rpc-duration) + RPC_DURATION="$2" + shift 2 + ;; + --tx-tps) + TX_TPS="$2" + shift 2 + ;; + --tx-duration) + TX_DURATION="$2" + shift 2 + ;; + --profile) + WORKLOAD_PROFILE="$2" + shift 2 + ;; + --with-benchmark) + WITH_BENCHMARK=true + shift + ;; + --skip-loki) + SKIP_LOKI=true + shift + ;; + --skip-regression) + SKIP_REGRESSION=true + shift + ;; + --cleanup) # Cleanup mode + log "Cleaning up..." + pkill -f "$WORKDIR" 2>/dev/null || true + docker compose -f "$COMPOSE_FILE" down 2>/dev/null || true + rm -rf "$WORKDIR" + ok "Cleanup complete." + exit 0 + ;; + -h | --help) usage ;; + *) die "Unknown option: $1" ;; + esac +done + +# --------------------------------------------------------------------------- +# Prerequisites +# --------------------------------------------------------------------------- +log "Checking prerequisites..." +[ -x "$XRPLD" ] || die "xrpld binary not found: $XRPLD" +command -v docker >/dev/null 2>&1 || die "docker not found" +docker compose version >/dev/null 2>&1 || die "docker compose (v2) not found" +command -v python3 >/dev/null 2>&1 || die "python3 not found" +command -v curl >/dev/null 2>&1 || die "curl not found" +command -v jq >/dev/null 2>&1 || die "jq not found" +[ -f "$COMPOSE_FILE" ] || die "docker-compose.workload.yaml not found" + +# Install Python dependencies. +log "Installing Python dependencies..." +pip3 install -q -r "$SCRIPT_DIR/requirements.txt" 2>/dev/null || + pip install -q -r "$SCRIPT_DIR/requirements.txt" 2>/dev/null || + warn "Could not install Python dependencies — they may already be present" + +ok "Prerequisites verified." + +# --------------------------------------------------------------------------- +# Cleanup previous run +# --------------------------------------------------------------------------- +log "Cleaning up previous run..." +pkill -f "$WORKDIR" 2>/dev/null || true +sleep 2 +rm -rf "$WORKDIR" +mkdir -p "$WORKDIR" "$REPORT_DIR" + +# --------------------------------------------------------------------------- +# Step 1: Start observability stack +# --------------------------------------------------------------------------- +log "Step 1: Starting observability stack..." +# Point the collector's log mount at this run's workdir so the filelog +# receiver tails the per-node debug.log files generated below. +XRPLD_LOG_DIR="$WORKDIR" docker compose -f "$COMPOSE_FILE" up -d + +log "Waiting for OTel Collector..." +for attempt in $(seq 1 30); do + status=$(curl -so /dev/null -w '%{http_code}' http://localhost:4318/ 2>/dev/null || echo 000) + if [ "$status" != "000" ]; then + ok "OTel Collector ready (attempt $attempt)" + break + fi + [ "$attempt" -eq 30 ] && die "OTel Collector not ready after 30s" + sleep 1 +done + +log "Waiting for Tempo..." +for attempt in $(seq 1 30); do + if curl -sf "http://localhost:3200/ready" >/dev/null 2>&1; then + ok "Tempo ready (attempt $attempt)" + break + fi + [ "$attempt" -eq 30 ] && die "Tempo not ready after 30s" + sleep 1 +done + +log "Waiting for Prometheus..." +for attempt in $(seq 1 30); do + if curl -sf "http://localhost:9090/-/healthy" >/dev/null 2>&1; then + ok "Prometheus ready (attempt $attempt)" + break + fi + [ "$attempt" -eq 30 ] && die "Prometheus not ready after 30s" + sleep 1 +done + +# --------------------------------------------------------------------------- +# Step 2: Generate validator keys and start cluster +# --------------------------------------------------------------------------- +log "Step 2: Starting $NUM_NODES-node validator cluster..." + +bash "$SCRIPT_DIR/generate-validator-keys.sh" "$XRPLD" "$NUM_NODES" "$WORKDIR" + +for i in $(seq 1 "$NUM_NODES"); do + NODE_DIR="$WORKDIR/node$i" + mkdir -p "$NODE_DIR/nudb" "$NODE_DIR/db" + + RPC_PORT=$((RPC_PORT_BASE + i - 1)) + WS_PORT=$((WS_PORT_BASE + i - 1)) + PEER_PORT=$((PEER_PORT_BASE + i - 1)) + SEED=$(jq -r ".[$((i - 1))].seed" "$WORKDIR/validator-keys.json") + + # Build ips_fixed. + IPS_FIXED="" + for j in $(seq 1 "$NUM_NODES"); do + if [ "$j" -ne "$i" ]; then + IPS_FIXED="${IPS_FIXED}127.0.0.1 $((PEER_PORT_BASE + j - 1)) +" + fi + done + + cat >"$NODE_DIR/xrpld.cfg" <"$NODE_DIR/stdout.log" 2>&1 & + echo $! >"$NODE_DIR/xrpld.pid" + log " Node $i: RPC=$RPC_PORT WS=$WS_PORT Peer=$PEER_PORT PID=$!" +done + +# --------------------------------------------------------------------------- +# Step 3: Wait for consensus +# --------------------------------------------------------------------------- +log "Step 3: Waiting for consensus..." +for attempt in $(seq 1 120); do + ready=0 + for i in $(seq 1 "$NUM_NODES"); do + port=$((RPC_PORT_BASE + i - 1)) + state=$(curl -sf "http://localhost:$port" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.server_state' 2>/dev/null || echo "") + if [ "$state" = "proposing" ]; then + ready=$((ready + 1)) + fi + done + if [ "$ready" -ge "$NUM_NODES" ]; then + ok "All $NUM_NODES nodes proposing (attempt $attempt)" + break + fi + if [ "$attempt" -eq 120 ]; then + warn "Consensus timeout — $ready/$NUM_NODES nodes ready" + fi + printf "\r %d/%d nodes proposing..." "$ready" "$NUM_NODES" + sleep 1 +done +echo "" + +# Wait for first validated ledger. +log "Waiting for validated ledger..." +for attempt in $(seq 1 60); do + val_seq=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + -d '{"method":"server_info"}' 2>/dev/null | + jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) + if [ "$val_seq" -gt 2 ] 2>/dev/null; then + ok "Validated ledger: seq $val_seq" + break + fi + [ "$attempt" -eq 60 ] && warn "No validated ledger after 60s" + sleep 1 +done + +# --------------------------------------------------------------------------- +# Step 4: Run workload orchestrator +# --------------------------------------------------------------------------- +log "Step 4: Running workload orchestrator (profile: $WORKLOAD_PROFILE)..." + +WS_ENDPOINTS="" +for i in $(seq 1 "$NUM_NODES"); do + WS_ENDPOINTS="$WS_ENDPOINTS ws://localhost:$((WS_PORT_BASE + i - 1))" +done + +python3 "$SCRIPT_DIR/workload_orchestrator.py" \ + --profile "$WORKLOAD_PROFILE" \ + --endpoints $WS_ENDPOINTS \ + --report "$REPORT_DIR/workload-report.json" \ + --report-dir "$REPORT_DIR" || + warn "Workload orchestrator returned non-zero exit" + +ok "Workload orchestration complete." + +# --------------------------------------------------------------------------- +# Step 5: Run telemetry validation suite +# --------------------------------------------------------------------------- +log "Step 5: Running telemetry validation suite..." + +VALIDATION_ARGS="--report $REPORT_DIR/validation-report.json" +if [ "$SKIP_LOKI" = true ]; then + VALIDATION_ARGS="$VALIDATION_ARGS --skip-loki" +fi + +VALIDATION_EXIT=0 +python3 "$SCRIPT_DIR/validate_telemetry.py" $VALIDATION_ARGS || VALIDATION_EXIT=$? + +if [ "$VALIDATION_EXIT" -eq 0 ]; then + ok "All telemetry validation checks passed!" +else + fail "Some telemetry validation checks failed (exit $VALIDATION_EXIT)" +fi + +# --------------------------------------------------------------------------- +# Step 6: Capture OTel timings and run the regression comparison +# --------------------------------------------------------------------------- +# This step ALWAYS captures timings (so CI always has an artifact from which +# to bootstrap/refresh the committed baseline). The comparator then either: +# - prints the paste-me JSON when the baseline is a placeholder, or +# - enforces thresholds and fails the run on regression. +# Use --skip-regression to opt out (e.g. for ad-hoc local exploration). +TIMINGS_FILE="$REPORT_DIR/timings.json" +REGRESSION_REPORT="$REPORT_DIR/regression-report.json" +REGRESSION_EXIT=0 + +if [ "$SKIP_REGRESSION" != true ]; then + log "Step 6: Capturing OTel timings from Prometheus..." + if python3 "$SCRIPT_DIR/capture_timings.py" \ + --prometheus "http://localhost:9090" \ + --metrics "$METRICS_FILE" \ + --output "$TIMINGS_FILE" \ + --window "$REGRESSION_WINDOW" \ + --profile "$WORKLOAD_PROFILE"; then + ok "Timings captured: $TIMINGS_FILE" + else + fail "Failed to capture timings — skipping regression comparison." + REGRESSION_EXIT=2 + SKIP_REGRESSION=true + fi +fi + +if [ "$SKIP_REGRESSION" != true ]; then + log "Comparing against baseline $BASELINE_FILE..." + python3 "$SCRIPT_DIR/compare_to_baseline.py" \ + --timings "$TIMINGS_FILE" \ + --baseline "$BASELINE_FILE" \ + --thresholds "$THRESHOLDS_FILE" \ + --report "$REGRESSION_REPORT" || REGRESSION_EXIT=$? + if [ "$REGRESSION_EXIT" -eq 0 ]; then + ok "Regression gate passed (or baseline placeholder — paste JSON printed above)." + elif [ "$REGRESSION_EXIT" -eq 1 ]; then + fail "Regression detected — see $REGRESSION_REPORT" + else + fail "Regression comparator internal error (exit $REGRESSION_EXIT)" + fi +else + warn "Regression gate skipped." +fi + +# --------------------------------------------------------------------------- +# Step 7: (Optional) Run overhead benchmark +# --------------------------------------------------------------------------- +if [ "$WITH_BENCHMARK" = true ]; then + log "Step 7: Running performance benchmark..." + bash "$SCRIPT_DIR/benchmark.sh" \ + --xrpld "$XRPLD" \ + --duration 120 \ + --nodes 3 \ + --output "$REPORT_DIR" || + warn "Benchmark returned non-zero exit" +fi + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +echo "" +echo "===========================================================" +echo " FULL VALIDATION RESULTS" +echo "===========================================================" +echo "" +echo " Reports directory: $REPORT_DIR" +echo "" +ls -la "$REPORT_DIR/" 2>/dev/null || true +echo "" +echo " Observability stack is running:" +echo " Tempo: http://localhost:3200" +echo " Grafana: http://localhost:3000" +echo " Prometheus: http://localhost:9090" +echo "" +echo " xrpld nodes ($NUM_NODES) are running:" +for i in $(seq 1 "$NUM_NODES"); do + rpc=$((RPC_PORT_BASE + i - 1)) + ws=$((WS_PORT_BASE + i - 1)) + pid=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null || echo 'unknown') + echo " Node $i: RPC=$rpc WS=$ws PID=$pid" +done +echo "" +echo " To tear down:" +echo " $0 --cleanup" +echo "" +echo "===========================================================" + +# Fail the run if EITHER validation or the regression gate failed. The +# `[ "$VAR" -gt N ]` comparison works here because exit codes are numeric. +FINAL_EXIT=0 +if [ "$VALIDATION_EXIT" -ne 0 ]; then + FINAL_EXIT="$VALIDATION_EXIT" +fi +if [ "$REGRESSION_EXIT" -ne 0 ] && [ "$FINAL_EXIT" -eq 0 ]; then + FINAL_EXIT="$REGRESSION_EXIT" +fi +exit "$FINAL_EXIT" diff --git a/docker/telemetry/workload/test_accounts.json b/docker/telemetry/workload/test_accounts.json new file mode 100644 index 00000000000..cb85670f52e --- /dev/null +++ b/docker/telemetry/workload/test_accounts.json @@ -0,0 +1,42 @@ +{ + "genesis": { + "account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", + "seed": "snoPBrXtMeMyMHUVTgbuqAfg1SUTb", + "description": "Genesis account with all XRP. Used to fund test accounts." + }, + "test_accounts": [ + { + "name": "alice", + "description": "Primary sender for Payment and OfferCreate transactions." + }, + { + "name": "bob", + "description": "Primary receiver for Payment transactions." + }, + { + "name": "carol", + "description": "TrustSet and issued currency counterparty." + }, + { + "name": "dave", + "description": "NFToken operations (mint, offer, accept)." + }, + { + "name": "eve", + "description": "Escrow operations (create, finish)." + }, + { + "name": "frank", + "description": "AMM pool operations (create, deposit, withdraw)." + }, + { + "name": "grace", + "description": "Additional sender for parallel transaction submission." + }, + { + "name": "heidi", + "description": "Additional receiver for payment diversity." + } + ], + "note": "Test account keypairs are generated dynamically at runtime via wallet_propose RPC. This file defines the logical roles. Actual keys are stored in the workdir during execution." +} diff --git a/docker/telemetry/workload/tx_submitter.py b/docker/telemetry/workload/tx_submitter.py new file mode 100644 index 00000000000..4ff015f40f1 --- /dev/null +++ b/docker/telemetry/workload/tx_submitter.py @@ -0,0 +1,848 @@ +#!/usr/bin/env python3 +"""Transaction Submitter for rippled telemetry validation. + +Generates diverse transaction types against a rippled cluster to exercise +the full span and metric surface: tx.process, tx.apply, ledger.build, +consensus.*, and all associated attributes. + +Pre-funds test accounts from the genesis account, then submits a +configurable mix of transaction types at a target TPS. + +Supported transaction types: + - Payment (XRP and issued currencies) + - OfferCreate / OfferCancel (DEX activity) + - TrustSet (trust line creation) + - NFTokenMint / NFTokenCreateOffer / NFTokenAcceptOffer + - EscrowCreate / EscrowFinish + - AMMCreate / AMMDeposit / AMMWithdraw (if amendment enabled) + +Usage: + python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120 + + # Custom transaction mix: + python3 tx_submitter.py --endpoint ws://localhost:6006 \\ + --weights '{"Payment":50,"OfferCreate":20,"TrustSet":10,"NFTokenMint":10,"EscrowCreate":10}' +""" + +import argparse +import asyncio +import json +import logging +import random +import sys +import time +from dataclasses import dataclass, field +from typing import Any + +import websockets + +logger = logging.getLogger("tx_submitter") + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +GENESIS_ACCOUNT = "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh" +GENESIS_SEED = "snoPBrXtMeMyMHUVTgbuqAfg1SUTb" + +# Amount to fund each test account (100,000 XRP in drops). +FUND_AMOUNT = "100000000000" + +# Default transaction mix weights (relative). +DEFAULT_TX_WEIGHTS: dict[str, int] = { + "Payment": 40, + "OfferCreate": 15, + "OfferCancel": 5, + "TrustSet": 10, + "NFTokenMint": 10, + "NFTokenCreateOffer": 5, + "EscrowCreate": 5, + "EscrowFinish": 5, + "AMMCreate": 3, + "AMMDeposit": 2, +} + +# Number of test accounts to create. +NUM_TEST_ACCOUNTS = 8 + + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclass +class Account: + """Represents a funded XRPL test account. + + Attributes: + name: Human-readable name (e.g., "alice"). + account: Classic address (rXXX...). + seed: Secret seed for signing. + sequence: Next available sequence number. + """ + + name: str + account: str + seed: str + sequence: int = 0 + + +@dataclass +class TxStats: + """Tracks transaction submission results. + + Attributes: + total_submitted: Total transactions sent to the network. + total_success: Transactions that returned tesSUCCESS or terQUEUED. + total_errors: Transactions that returned an error engine_result. + by_type: Per-transaction-type count of submissions. + errors_by_type: Per-transaction-type count of errors. + """ + + total_submitted: int = 0 + total_success: int = 0 + total_errors: int = 0 + by_type: dict[str, int] = field(default_factory=dict) + errors_by_type: dict[str, int] = field(default_factory=dict) + + def record(self, tx_type: str, success: bool) -> None: + """Record the result of a transaction submission.""" + self.total_submitted += 1 + self.by_type[tx_type] = self.by_type.get(tx_type, 0) + 1 + if success: + self.total_success += 1 + else: + self.total_errors += 1 + self.errors_by_type[tx_type] = self.errors_by_type.get(tx_type, 0) + 1 + + def summary(self) -> dict[str, Any]: + """Return a summary dict suitable for JSON serialization.""" + return { + "total_submitted": self.total_submitted, + "total_success": self.total_success, + "total_errors": self.total_errors, + "success_rate_pct": ( + round(self.total_success / self.total_submitted * 100, 2) + if self.total_submitted + else 0 + ), + "by_type": self.by_type, + "errors_by_type": self.errors_by_type, + } + + +# --------------------------------------------------------------------------- +# WebSocket RPC helpers +# --------------------------------------------------------------------------- + + +async def ws_request( + ws: websockets.WebSocketClientProtocol, + command: str, + params: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Send a native WebSocket command and return the result payload. + + Uses rippled's native WebSocket format (``command`` key with flat + parameters). The response has ``status`` at the top level and the + actual data payload inside ``result``. This helper unwraps the + ``result`` dict so callers can read fields directly. + + Args: + ws: Open WebSocket connection. + command: RPC command name (e.g., ``account_info``, ``submit``). + params: Optional flat parameter dict merged into the request. + + Returns: + The inner ``result`` dict from the response. + + Raises: + RuntimeError: If the request fails or times out. + """ + request: dict[str, Any] = {"command": command} + if params: + request.update(params) + await ws.send(json.dumps(request)) + raw = await asyncio.wait_for(ws.recv(), timeout=30.0) + resp = json.loads(raw) + + # WS command format: {"status": "success", "result": {...}, "type": "response"} + # On error: {"status": "error", "error": "...", "error_message": "..."} + if resp.get("status") == "error": + logger.warning( + "%s error: %s — %s", + command, + resp.get("error", "unknown"), + resp.get("error_message", ""), + ) + return resp.get("result", resp) + + +async def create_account(ws: websockets.WebSocketClientProtocol, name: str) -> Account: + """Create a new account via wallet_propose RPC. + + Args: + ws: Open WebSocket connection. + name: Human-readable name for the account. + + Returns: + An Account instance with the generated keypair. + """ + result = await ws_request(ws, "wallet_propose") + if "account_id" not in result: + raise RuntimeError( + f"wallet_propose failed: {json.dumps(result, indent=None)[:300]}" + ) + return Account( + name=name, + account=result["account_id"], + seed=result["master_seed"], + ) + + +async def fund_account( + ws: websockets.WebSocketClientProtocol, + dest: Account, + genesis_seq: int, +) -> tuple[bool, int]: + """Fund a test account from genesis. + + Args: + ws: Open WebSocket connection. + dest: Destination account to fund. + genesis_seq: Current genesis account sequence number. + + Returns: + Tuple of (success: bool, next_sequence: int). + """ + resp = await ws_request( + ws, + "submit", + { + "secret": GENESIS_SEED, + "tx_json": { + "TransactionType": "Payment", + "Account": GENESIS_ACCOUNT, + "Destination": dest.account, + "Amount": FUND_AMOUNT, + "Sequence": genesis_seq, + }, + }, + ) + engine_result = resp.get("engine_result", "unknown") + success = engine_result in ("tesSUCCESS", "terQUEUED") + if not success: + # Log the full response to help diagnose submit failures in CI. + logger.warning( + "Fund %s failed: engine_result=%s, full response: %s", + dest.name, + engine_result, + json.dumps(resp, indent=None)[:500], + ) + return success, genesis_seq + 1 + + +async def get_account_sequence( + ws: websockets.WebSocketClientProtocol, account: str +) -> int: + """Get the current sequence number for an account. + + Args: + ws: Open WebSocket connection. + account: Classic address. + + Returns: + Current sequence number. + """ + resp = await ws_request(ws, "account_info", {"account": account}) + if "account_data" not in resp: + # Log full response to diagnose WS API format issues. + logger.warning( + "account_info for %s: no account_data, full response: %s", + account[:12], + json.dumps(resp, indent=None)[:500], + ) + return 0 + return resp["account_data"].get("Sequence", 0) + + +# --------------------------------------------------------------------------- +# Transaction builders +# --------------------------------------------------------------------------- + + +def build_payment(sender: Account, receiver: Account) -> dict[str, Any]: + """Build an XRP Payment transaction. + + Args: + sender: Source account. + receiver: Destination account. + + Returns: + Transaction JSON and signing secret. + """ + amount = str(random.randint(1000, 1000000)) # 0.001 - 1 XRP + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "Payment", + "Account": sender.account, + "Destination": receiver.account, + "Amount": amount, + "Sequence": sender.sequence, + }, + } + + +def build_offer_create(sender: Account) -> dict[str, Any]: + """Build an OfferCreate transaction (XRP/USD pair). + + Args: + sender: Account placing the offer. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "OfferCreate", + "Account": sender.account, + "TakerPays": str(random.randint(100000, 10000000)), + "TakerGets": { + "currency": "USD", + "issuer": GENESIS_ACCOUNT, + "value": str(round(random.uniform(0.1, 100.0), 2)), + }, + "Sequence": sender.sequence, + }, + } + + +def build_offer_cancel(sender: Account) -> dict[str, Any]: + """Build an OfferCancel transaction. + + Uses a non-existent offer sequence — will fail gracefully but still + exercises the tx.process span pipeline. + + Args: + sender: Account cancelling the offer. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "OfferCancel", + "Account": sender.account, + "OfferSequence": max(1, sender.sequence - 1), + "Sequence": sender.sequence, + }, + } + + +def build_trust_set(sender: Account) -> dict[str, Any]: + """Build a TrustSet transaction for a USD trust line. + + Args: + sender: Account setting the trust line. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "TrustSet", + "Account": sender.account, + "LimitAmount": { + "currency": "USD", + "issuer": GENESIS_ACCOUNT, + "value": "1000000", + }, + "Sequence": sender.sequence, + }, + } + + +def build_nftoken_mint(sender: Account) -> dict[str, Any]: + """Build an NFTokenMint transaction. + + Args: + sender: Account minting the NFT. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "NFTokenMint", + "Account": sender.account, + "NFTokenTaxon": random.randint(0, 100), + "Flags": 8, # tfTransferable + "Sequence": sender.sequence, + }, + } + + +def build_nftoken_create_offer(sender: Account) -> dict[str, Any]: + """Build an NFTokenCreateOffer transaction. + + Uses a dummy NFTokenID — will fail but exercises the span pipeline. + + Args: + sender: Account creating the NFT offer. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "NFTokenCreateOffer", + "Account": sender.account, + "NFTokenID": "0" * 64, + "Amount": str(random.randint(100000, 1000000)), + "Flags": 1, # tfSellNFToken + "Sequence": sender.sequence, + }, + } + + +def build_escrow_create(sender: Account, receiver: Account) -> dict[str, Any]: + """Build an EscrowCreate transaction. + + Creates a time-based escrow that finishes 10 seconds from now. + + Args: + sender: Account creating the escrow. + receiver: Destination account for escrow funds. + + Returns: + Transaction JSON and signing secret. + """ + # Ripple epoch offset: 946684800 seconds from Unix epoch + ripple_time = int(time.time()) - 946684800 + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "EscrowCreate", + "Account": sender.account, + "Destination": receiver.account, + "Amount": str(random.randint(100000, 1000000)), + "FinishAfter": ripple_time + 10, + "Sequence": sender.sequence, + }, + } + + +def build_escrow_finish(sender: Account, owner: Account) -> dict[str, Any]: + """Build an EscrowFinish transaction. + + Uses a dummy offer sequence — will likely fail but exercises spans. + + Args: + sender: Account finishing the escrow. + owner: Account that created the escrow. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "EscrowFinish", + "Account": sender.account, + "Owner": owner.account, + "OfferSequence": max(1, owner.sequence - 2), + "Sequence": sender.sequence, + }, + } + + +def build_amm_create(sender: Account) -> dict[str, Any]: + """Build an AMMCreate transaction (XRP/USD pool). + + Requires the AMM amendment to be enabled on the network. + + Args: + sender: Account creating the AMM pool. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "AMMCreate", + "Account": sender.account, + "Amount": str(random.randint(10000000, 100000000)), + "Amount2": { + "currency": "USD", + "issuer": GENESIS_ACCOUNT, + "value": str(round(random.uniform(10.0, 1000.0), 2)), + }, + "TradingFee": 500, # 0.5% + "Sequence": sender.sequence, + }, + } + + +def build_amm_deposit(sender: Account) -> dict[str, Any]: + """Build an AMMDeposit transaction. + + Args: + sender: Account depositing into the AMM pool. + + Returns: + Transaction JSON and signing secret. + """ + return { + "secret": sender.seed, + "tx_json": { + "TransactionType": "AMMDeposit", + "Account": sender.account, + "Asset": {"currency": "XRP"}, + "Asset2": { + "currency": "USD", + "issuer": GENESIS_ACCOUNT, + }, + "Amount": str(random.randint(1000000, 10000000)), + "Flags": 0x00080000, # tfSingleAsset + "Sequence": sender.sequence, + }, + } + + +# Transaction type -> builder function mapping. +# Each builder takes (accounts: list[Account]) and returns submit params. +TX_BUILDERS: dict[str, Any] = { + "Payment": lambda accts: build_payment(accts[0], accts[1]), + "OfferCreate": lambda accts: build_offer_create(accts[0]), + "OfferCancel": lambda accts: build_offer_cancel(accts[0]), + "TrustSet": lambda accts: build_trust_set(accts[2]), + "NFTokenMint": lambda accts: build_nftoken_mint(accts[3]), + "NFTokenCreateOffer": lambda accts: build_nftoken_create_offer(accts[3]), + "EscrowCreate": lambda accts: build_escrow_create(accts[4], accts[1]), + "EscrowFinish": lambda accts: build_escrow_finish(accts[4], accts[4]), + "AMMCreate": lambda accts: build_amm_create(accts[5]), + "AMMDeposit": lambda accts: build_amm_deposit(accts[5]), +} + + +# --------------------------------------------------------------------------- +# Main submission loop +# --------------------------------------------------------------------------- + + +async def setup_accounts( + ws: websockets.WebSocketClientProtocol, +) -> list[Account]: + """Create and fund test accounts from genesis. + + Generates NUM_TEST_ACCOUNTS accounts via wallet_propose, then funds + each with FUND_AMOUNT XRP from genesis. + + Args: + ws: Open WebSocket connection to a rippled node. + + Returns: + List of funded Account instances. + """ + account_names = ["alice", "bob", "carol", "dave", "eve", "frank", "grace", "heidi"] + + logger.info("Creating %d test accounts...", NUM_TEST_ACCOUNTS) + accounts: list[Account] = [] + for name in account_names[:NUM_TEST_ACCOUNTS]: + acct = await create_account(ws, name) + accounts.append(acct) + logger.info(" Created %s: %s", name, acct.account) + + # Get genesis sequence. + genesis_seq = await get_account_sequence(ws, GENESIS_ACCOUNT) + logger.info("Genesis sequence: %d", genesis_seq) + + # Fund all accounts. + logger.info("Funding test accounts...") + for acct in accounts: + success, genesis_seq = await fund_account(ws, acct, genesis_seq) + if success: + logger.info(" Funded %s", acct.name) + else: + logger.warning(" Failed to fund %s", acct.name) + + # Wait for funding transactions to be validated. + logger.info("Waiting 10s for funding transactions to validate...") + await asyncio.sleep(10) + + # Refresh sequence numbers for all accounts. + for acct in accounts: + try: + acct.sequence = await get_account_sequence(ws, acct.account) + logger.info(" %s sequence: %d", acct.name, acct.sequence) + except Exception as exc: + logger.warning(" Failed to get sequence for %s: %s", acct.name, exc) + + return accounts + + +async def submit_transaction( + ws: websockets.WebSocketClientProtocol, + tx_type: str, + accounts: list[Account], + stats: TxStats, +) -> None: + """Submit a single transaction of the given type. + + Selects the appropriate builder, constructs the transaction, submits + it via the submit RPC, and records the result. + + Args: + ws: Open WebSocket connection. + tx_type: Transaction type name (e.g., "Payment"). + accounts: List of funded test accounts. + stats: TxStats instance to record results. + """ + builder = TX_BUILDERS.get(tx_type) + if not builder: + logger.warning("Unknown transaction type: %s", tx_type) + return + + try: + params = builder(accounts) + # Identify which account is the sender to bump its sequence. + sender_addr = params["tx_json"]["Account"] + sender = next((a for a in accounts if a.account == sender_addr), None) + + resp = await ws_request(ws, "submit", params) + engine_result = resp.get("engine_result", "unknown") + success = engine_result in ( + "tesSUCCESS", + "terQUEUED", + "tecUNFUNDED_OFFER", + "tecNO_DST_INSUF_XRP", + ) + stats.record(tx_type, success) + + if sender: + sender.sequence += 1 + + if not success: + logger.debug( + "%s result: %s (%s)", + tx_type, + engine_result, + resp.get("engine_result_message", ""), + ) + except Exception as exc: + stats.record(tx_type, False) + logger.debug("%s error: %s", tx_type, exc) + + +async def _refresh_sequences( + ws: websockets.WebSocketClientProtocol, + accounts: list[Account], +) -> None: + """Re-sync account sequences from the validated ledger. + + In a consensus network, other nodes' transactions advance sequences + beyond the submitter's local tracking. Refreshing every ~10 s keeps + the local counter close to the ledger and prevents tefPAST_SEQ storms. + """ + for acct in accounts: + try: + seq = await get_account_sequence(ws, acct.account) + if seq > acct.sequence: + acct.sequence = seq + except Exception: + pass + + +async def run_submitter( + endpoint: str, + tps: float, + duration: float, + weights: dict[str, int], +) -> TxStats: + """Run the transaction submitter against a single endpoint. + + Args: + endpoint: WebSocket URL (ws://host:port). + tps: Target transactions per second. + duration: Total run time in seconds. + weights: Transaction type distribution weights. + + Returns: + TxStats with aggregated results. + """ + stats = TxStats() + interval = 1.0 / tps if tps > 0 else 0.5 + + ws = await websockets.connect(endpoint, ping_interval=20, ping_timeout=10) + logger.info("Connected to %s", endpoint) + + try: + # Setup test accounts. + accounts = await setup_accounts(ws) + if len(accounts) < 6: + logger.error("Need at least 6 funded accounts, got %d", len(accounts)) + return stats + + # Build weighted command list. + tx_types = list(weights.keys()) + tx_weights = [weights[t] for t in tx_types] + + logger.info( + "Starting TX submission: tps=%s, duration=%ss, types=%d", + tps, + duration, + len(tx_types), + ) + + start = time.monotonic() + last_seq_refresh = start + seq_refresh_interval = 10.0 + while (time.monotonic() - start) < duration: + # Periodically re-sync account sequences from the ledger so + # locally-tracked sequences don't drift behind consensus. + if (time.monotonic() - last_seq_refresh) >= seq_refresh_interval: + await _refresh_sequences(ws, accounts) + last_seq_refresh = time.monotonic() + + tx_type = random.choices(tx_types, weights=tx_weights, k=1)[0] + await submit_transaction(ws, tx_type, accounts, stats) + await asyncio.sleep(interval) + + # Progress logging every 50 transactions. + if stats.total_submitted % 50 == 0 and stats.total_submitted > 0: + elapsed = time.monotonic() - start + actual_tps = stats.total_submitted / elapsed if elapsed > 0 else 0 + logger.info( + "Progress: %d submitted, %d success, %d errors, " + "%.1f TPS (%.0fs elapsed)", + stats.total_submitted, + stats.total_success, + stats.total_errors, + actual_tps, + elapsed, + ) + + finally: + await ws.close() + + elapsed = time.monotonic() - start + logger.info( + "Submission complete: %d submitted, %d success, %d errors " + "in %.1fs (%.1f TPS)", + stats.total_submitted, + stats.total_success, + stats.total_errors, + elapsed, + stats.total_submitted / elapsed if elapsed > 0 else 0, + ) + + return stats + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Transaction Submitter for rippled telemetry validation", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Basic usage (5 TPS for 2 minutes): + python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120 + + # Custom transaction mix: + python3 tx_submitter.py --endpoint ws://localhost:6006 \\ + --weights '{"Payment": 60, "OfferCreate": 20, "TrustSet": 20}' + """, + ) + parser.add_argument( + "--endpoint", + type=str, + default="ws://localhost:6006", + help="WebSocket endpoint (default: ws://localhost:6006)", + ) + parser.add_argument( + "--tps", + type=float, + default=5.0, + help="Target transactions per second (default: 5)", + ) + parser.add_argument( + "--duration", + type=float, + default=120.0, + help="Run duration in seconds (default: 120)", + ) + parser.add_argument( + "--weights", + type=str, + default=None, + help="JSON string of transaction type weights (overrides defaults)", + ) + parser.add_argument( + "--output", + type=str, + default=None, + help="Write JSON summary to this file path", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable debug logging", + ) + return parser.parse_args() + + +def main() -> None: + """Main entry point for the transaction submitter.""" + args = parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s %(message)s", + ) + + # Parse custom weights if provided. + weights = DEFAULT_TX_WEIGHTS.copy() + if args.weights: + try: + custom = json.loads(args.weights) + weights = {k: int(v) for k, v in custom.items()} + logger.info("Using custom weights: %s", weights) + except (json.JSONDecodeError, ValueError) as exc: + logger.error("Invalid --weights JSON: %s", exc) + sys.exit(1) + + # Run the submitter. + stats = asyncio.run( + run_submitter( + endpoint=args.endpoint, + tps=args.tps, + duration=args.duration, + weights=weights, + ) + ) + + summary = stats.summary() + print(json.dumps(summary, indent=2)) + + if args.output: + with open(args.output, "w") as f: + json.dump(summary, f, indent=2) + logger.info("Summary written to %s", args.output) + + +if __name__ == "__main__": + main() diff --git a/docker/telemetry/workload/validate_telemetry.py b/docker/telemetry/workload/validate_telemetry.py new file mode 100644 index 00000000000..40f7e507a82 --- /dev/null +++ b/docker/telemetry/workload/validate_telemetry.py @@ -0,0 +1,1321 @@ +#!/usr/bin/env python3 +"""Telemetry Validation Suite for xrpld. + +Validates that the full telemetry stack is emitting expected data after +a workload run. Queries Tempo (spans), Prometheus (metrics), Loki (logs), +and Grafana (dashboards) APIs to produce a pass/fail report. + +Validation categories: + 1. Span validation — All 16+ span types present with required attributes + 2. Metric validation — SpanMetrics, StatsD, and Phase 9 metrics are non-zero + 3. Log-trace correlation — Loki logs contain trace_id/span_id fields + 4. Dashboard validation — All 14 Grafana dashboards render data + 5. External parity — Span attrs, metric existence, and value sanity for + external dashboard parity (validator-health, + peer-quality, node-health) + +Usage: + python3 validate_telemetry.py --report /tmp/validation-report.json + + # Custom API endpoints: + python3 validate_telemetry.py \\ + --tempo http://localhost:3200 \\ + --prometheus http://localhost:9090 \\ + --loki http://localhost:3100 \\ + --grafana http://localhost:3000 +""" + +import argparse +import asyncio +import json +import logging +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import aiohttp + +logger = logging.getLogger("validate_telemetry") + +# --------------------------------------------------------------------------- +# Configuration defaults +# --------------------------------------------------------------------------- + +DEFAULT_TEMPO = "http://localhost:3200" +DEFAULT_PROMETHEUS = "http://localhost:9090" +DEFAULT_LOKI = "http://localhost:3100" +DEFAULT_GRAFANA = "http://localhost:3000" + +SCRIPT_DIR = Path(__file__).parent +EXPECTED_SPANS_FILE = SCRIPT_DIR / "expected_spans.json" +EXPECTED_METRICS_FILE = SCRIPT_DIR / "expected_metrics.json" + +# Some beast::insight gauges/counters (ledger-age, peer-finder, overlay +# traffic) only populate after the node validates ledgers and sustains peer +# traffic, then travel a 1s periodic OTLP export + a 15s Prometheus scrape +# before they are queryable. On a slow CI runner the fixed post-workload wait +# can end before that pipeline settles, so a single query races and reports 0 +# series. Poll each missing metric for up to this long (covering two scrape +# cycles) before failing, so the check is robust to runner speed. +METRIC_POLL_TIMEOUT_SEC = 45.0 +METRIC_POLL_INTERVAL_SEC = 5.0 + + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclass +class CheckResult: + """Result of a single validation check. + + Attributes: + name: Check identifier (e.g., "span.rpc.ws_message"). + category: Validation category (span, metric, log, dashboard). + passed: Whether the check passed. + message: Human-readable description of the result. + details: Optional additional data (counts, values, etc.). + """ + + name: str + category: str + passed: bool + message: str + details: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + """Serialize to a JSON-compatible dict.""" + return { + "name": self.name, + "category": self.category, + "passed": self.passed, + "message": self.message, + "details": self.details, + } + + +@dataclass +class ValidationReport: + """Aggregated validation report. + + Attributes: + checks: List of all individual check results. + start_time: ISO timestamp when validation started. + end_time: ISO timestamp when validation completed. + """ + + checks: list[CheckResult] = field(default_factory=list) + start_time: str = "" + end_time: str = "" + + @property + def total_checks(self) -> int: + """Total number of checks executed.""" + return len(self.checks) + + @property + def passed(self) -> int: + """Number of checks that passed.""" + return sum(1 for c in self.checks if c.passed) + + @property + def failed(self) -> int: + """Number of checks that failed.""" + return sum(1 for c in self.checks if not c.passed) + + @property + def all_passed(self) -> bool: + """Whether all checks passed.""" + return self.failed == 0 + + def add(self, check: CheckResult) -> None: + """Add a check result to the report.""" + self.checks.append(check) + status = "PASS" if check.passed else "FAIL" + logger.info("[%s] %s: %s", status, check.name, check.message) + + def to_dict(self) -> dict[str, Any]: + """Serialize to a JSON-compatible dict.""" + return { + "summary": { + "total": self.total_checks, + "passed": self.passed, + "failed": self.failed, + "all_passed": self.all_passed, + }, + "start_time": self.start_time, + "end_time": self.end_time, + "checks": [c.to_dict() for c in self.checks], + } + + +# --------------------------------------------------------------------------- +# Tempo API helpers +# --------------------------------------------------------------------------- + + +async def _tempo_search( + session: aiohttp.ClientSession, + tempo_url: str, + query: str, + limit: int = 20, +) -> list[dict[str, Any]]: + """Search traces in Tempo using TraceQL. + + Args: + session: aiohttp client session. + tempo_url: Base URL for Tempo API (e.g., http://localhost:3200). + query: TraceQL query string. + limit: Maximum number of traces to return. + + Returns: + List of trace summary dicts from Tempo search results. + """ + params = {"q": query, "limit": str(limit)} + async with session.get(f"{tempo_url}/api/search", params=params) as resp: + data = await resp.json() + return data.get("traces", []) + + +async def _tempo_get_trace( + session: aiohttp.ClientSession, + tempo_url: str, + trace_id: str, +) -> list[dict[str, Any]]: + """Fetch a full trace from Tempo by trace ID. + + Returns the list of spans extracted from the OTLP-format response. + + Args: + session: aiohttp client session. + tempo_url: Tempo API base URL. + trace_id: Hex trace ID string. + + Returns: + Flat list of span dicts with 'name' and 'attributes' keys. + """ + async with session.get(f"{tempo_url}/api/traces/{trace_id}") as resp: + data = await resp.json() + spans: list[dict[str, Any]] = [] + for batch in data.get("batches", []): + for scope_spans in batch.get("scopeSpans", []): + spans.extend(scope_spans.get("spans", [])) + return spans + + +def _otlp_span_attr_keys(span: dict[str, Any]) -> set[str]: + """Extract all attribute key names from an OTLP span. + + Args: + span: OTLP span dict with an 'attributes' list. + + Returns: + Set of attribute key strings. + """ + return {a["key"] for a in span.get("attributes", []) if "key" in a} + + +# --------------------------------------------------------------------------- +# Span Validation (Tempo API) +# --------------------------------------------------------------------------- + + +async def validate_spans( + session: aiohttp.ClientSession, + tempo_url: str, + report: ValidationReport, +) -> None: + """Validate that all expected spans appear in Tempo. + + Queries the Tempo TraceQL API for each expected span name and checks + that traces exist. Also validates required attributes on spans and + parent-child relationships. + + Args: + session: aiohttp client session. + tempo_url: Base URL for Tempo API (e.g., http://localhost:3200). + report: ValidationReport to accumulate results. + """ + logger.info("--- Span Validation (Tempo) ---") + + # Load expected spans. + with open(EXPECTED_SPANS_FILE) as f: + expected = json.load(f) + + # Check service registration. + try: + async with session.get( + f"{tempo_url}/api/v2/search/tag/resource.service.name/values" + ) as resp: + data = await resp.json() + tag_values = data.get("tagValues", []) + services = [tv.get("value", "") for tv in tag_values] + has_xrpld = "xrpld" in services + report.add( + CheckResult( + name="span.service_registration", + category="span", + passed=has_xrpld, + message=( + f"Service 'xrpld' registered (found: {services})" + if has_xrpld + else f"Service 'xrpld' NOT found (found: {services})" + ), + ) + ) + except Exception as exc: + report.add( + CheckResult( + name="span.service_registration", + category="span", + passed=False, + message=f"Tempo API unreachable: {exc}", + ) + ) + return + + # Diagnostic: list all available operations (span names) for the xrpld + # service. This output appears in CI logs and helps debug missing-span + # failures without needing to reproduce the full stack locally. + try: + async with session.get( + f"{tempo_url}/api/v2/search/tag/span.name/values" + ) as resp: + ops_data = await resp.json() + tag_values = ops_data.get("tagValues", []) + operations = [tv.get("value", "") for tv in tag_values] + logger.info( + "Tempo operations (%d total): %s", + len(operations), + operations, + ) + except Exception as exc: + logger.warning("Failed to fetch Tempo operations: %s", exc) + + # Concrete probe names for wildcard span entries. Exact-match TraceQL can't + # match a literal "*", so a representative operation name is substituted. + # Wildcards without a known concrete example (e.g. grpc. when no + # gRPC client runs) are skipped when marked optional. + wildcard_probes = {"rpc.command.*": "rpc.command.server_info"} + + # Check each expected span. + for span_def in expected["spans"]: + span_name = span_def["name"] + is_optional = span_def.get("optional", False) + check_name = f"span.{span_name}" + + if "*" in span_name: + operation = wildcard_probes.get(span_name) + if operation is None: + # No concrete probe. Optional wildcards (e.g. grpc.*) are skipped; + # a required one would be a config error worth surfacing. + if is_optional: + logger.info( + "[SKIP] %s: optional wildcard span with no concrete " + "probe (not exercised by the workload)", + check_name, + ) + continue + report.add( + CheckResult( + name=check_name, + category="span", + passed=False, + message=f"{span_name}: required wildcard has no probe name", + ) + ) + continue + else: + operation = span_name + + try: + query = '{resource.service.name="xrpld" && name="' + operation + '"}' + traces = await _tempo_search(session, tempo_url, query, limit=5) + count = len(traces) + # Optional spans only fire under specific traffic (mode changes, + # missing-ledger fetch, fee escalation). Absence is not a failure — + # mirror the parent-child "skip" handling so CI stays green. + if count == 0 and is_optional: + logger.info( + "[SKIP] %s: optional span not emitted under this workload", + check_name, + ) + report.add( + CheckResult( + name=check_name, + category="span", + passed=True, + message=f"{span_name}: optional, not emitted (skipped)", + details={"trace_count": 0, "optional": True}, + ) + ) + continue + report.add( + CheckResult( + name=check_name, + category="span", + passed=count > 0, + message=( + f"{span_name}: {count} traces found" + if count > 0 + else f"{span_name}: 0 traces (expected > 0)" + ), + details={"trace_count": count}, + ) + ) + + # Validate required attributes on first trace. + if count > 0 and span_def.get("required_attributes"): + trace_id = traces[0].get("traceID", "") + if trace_id: + spans = await _tempo_get_trace(session, tempo_url, trace_id) + await _validate_span_attributes_otlp(spans, span_def, report) + except Exception as exc: + report.add( + CheckResult( + name=check_name, + category="span", + passed=False, + message=f"{span_name}: query failed ({exc})", + ) + ) + + # Validate parent-child relationships. + for rel in expected.get("parent_child_relationships", []): + # Skip relationships marked with "skip: true" (e.g., cross-thread + # parent-child that requires a C++ fix to propagate span context). + if rel.get("skip", False): + reason = rel.get("skip_reason", "marked skip in expected_spans.json") + logger.info( + "[SKIP] span.hierarchy.%s->%s: %s", + rel["parent"], + rel["child"], + reason, + ) + continue + await _validate_parent_child(session, tempo_url, rel, report) + + +async def _validate_span_attributes_otlp( + spans: list[dict[str, Any]], + span_def: dict[str, Any], + report: ValidationReport, +) -> None: + """Check that OTLP spans contain expected attributes. + + Args: + spans: List of OTLP span dicts from Tempo. + span_def: Span definition from expected_spans.json. + report: ValidationReport to accumulate results. + """ + required_attrs = span_def.get("required_attributes", []) + if not required_attrs: + return + + span_name = span_def["name"] + # Collect all attribute keys from all spans. + found_attrs: set[str] = set() + for span in spans: + found_attrs.update(_otlp_span_attr_keys(span)) + + missing = [a for a in required_attrs if a not in found_attrs] + report.add( + CheckResult( + name=f"span.attrs.{span_name}", + category="span", + passed=len(missing) == 0, + message=( + f"{span_name}: all {len(required_attrs)} attributes present" + if not missing + else f"{span_name}: missing attributes: {missing}" + ), + details={ + "required": required_attrs, + "found": list(found_attrs), + "missing": missing, + }, + ) + ) + + +async def _validate_parent_child( + session: aiohttp.ClientSession, + tempo_url: str, + relationship: dict[str, Any], + report: ValidationReport, +) -> None: + """Validate a parent-child span relationship in Tempo traces. + + Args: + session: aiohttp client session. + tempo_url: Base URL for Tempo API. + relationship: Dict with 'parent' and 'child' span names. + report: ValidationReport to accumulate results. + """ + parent_name = relationship["parent"] + child_name = relationship["child"] + + try: + # Query traces for the parent span. + query = '{resource.service.name="xrpld" && name="' + parent_name + '"}' + traces = await _tempo_search(session, tempo_url, query, limit=3) + + if not traces: + report.add( + CheckResult( + name=f"span.hierarchy.{parent_name}->{child_name}", + category="span", + passed=False, + message=f"No {parent_name} traces to check hierarchy", + ) + ) + return + + # Check if child spans exist within parent traces. + # Use the concrete child name for wildcard patterns. + concrete_child = child_name.replace("*", "server_info") + found_child = False + for trace_summary in traces: + trace_id = trace_summary.get("traceID", "") + if not trace_id: + continue + spans = await _tempo_get_trace(session, tempo_url, trace_id) + for span in spans: + op = span.get("name", "") + if concrete_child in op or ("*" not in child_name and op == child_name): + found_child = True + break + if found_child: + break + + report.add( + CheckResult( + name=f"span.hierarchy.{parent_name}->{child_name}", + category="span", + passed=found_child, + message=( + f"Found {child_name} as child of {parent_name}" + if found_child + else f"{child_name} not found in {parent_name} traces" + ), + ) + ) + except Exception as exc: + report.add( + CheckResult( + name=f"span.hierarchy.{parent_name}->{child_name}", + category="span", + passed=False, + message=f"Hierarchy check failed: {exc}", + ) + ) + + +# --------------------------------------------------------------------------- +# Metric Validation (Prometheus API) +# --------------------------------------------------------------------------- + + +async def validate_metrics( + session: aiohttp.ClientSession, + prometheus_url: str, + report: ValidationReport, +) -> None: + """Validate that expected metrics appear in Prometheus with non-zero values. + + Args: + session: aiohttp client session. + prometheus_url: Base URL for Prometheus API (e.g., http://localhost:9090). + report: ValidationReport to accumulate results. + """ + logger.info("--- Metric Validation (Prometheus) ---") + + # Diagnostic: list all metric names in Prometheus. Helps debug name + # mismatches between expected_metrics.json and actual emissions. + try: + async with session.get( + f"{prometheus_url}/api/v1/label/__name__/values" + ) as resp: + label_data = await resp.json() + all_metrics = label_data.get("data", []) + # Log relevant metrics for debugging. + relevant = [ + m + for m in all_metrics + if m.startswith( + ( + "span_", + "rpc_method", + "cache_", + "txq_", + "object_count", + "load_factor", + "nodestore", + "ledgermaster", + "peer_finder", + "jobq_", + "total_bytes", + "total_messages", + "validation_agreement", + "validator_health", + "peer_quality", + "ledger_economy", + "state_tracking", + "storage_detail", + ) + ) + ] + logger.info( + "Prometheus metrics (relevant, %d of %d total): %s", + len(relevant), + len(all_metrics), + relevant, + ) + except Exception as exc: + logger.warning("Failed to fetch Prometheus metric names: %s", exc) + + with open(EXPECTED_METRICS_FILE) as f: + expected = json.load(f) + + # Check each metric category. + for category_key, category_data in expected.items(): + if category_key in ("description", "grafana_dashboards"): + continue + + metrics = category_data.get("metrics", []) + for metric_name in metrics: + await _check_prometheus_metric( + session, prometheus_url, metric_name, category_key, report + ) + + +async def _check_prometheus_metric( + session: aiohttp.ClientSession, + prometheus_url: str, + metric_name: str, + category: str, + report: ValidationReport, +) -> None: + """Query Prometheus for a specific metric and check it exists. + + Args: + session: aiohttp client session. + prometheus_url: Prometheus base URL. + metric_name: Prometheus metric name. + category: Metric category for the report. + report: ValidationReport to accumulate results. + """ + try: + # Use the /api/v1/series endpoint instead of an instant query. + # Beast::insight StatsD gauges only mark dirty on value *changes*, + # so a gauge that stabilizes (e.g. peer count stays at 1) may go + # stale in Prometheus and disappear from instant queries. The + # series endpoint returns any metric that existed in the window, + # regardless of staleness. + # + # Poll rather than query once: late-populating gauges/counters may + # not have completed the export+scrape pipeline when this runs, so a + # single query races. Re-query until the metric appears or the poll + # window elapses; a metric that never appears still fails after the + # timeout. + params: dict[str, str] = {"match[]": metric_name} + series_count = 0 + deadline = time.monotonic() + METRIC_POLL_TIMEOUT_SEC + while True: + async with session.get( + f"{prometheus_url}/api/v1/series", params=params + ) as resp: + data = await resp.json() + series_count = len(data.get("data", [])) + if series_count > 0 or time.monotonic() >= deadline: + break + await asyncio.sleep(METRIC_POLL_INTERVAL_SEC) + report.add( + CheckResult( + name=f"metric.{category}.{metric_name}", + category="metric", + passed=series_count > 0, + message=( + f"{metric_name}: {series_count} series" + if series_count > 0 + else f"{metric_name}: 0 series (expected > 0)" + ), + details={"series_count": series_count}, + ) + ) + except Exception as exc: + report.add( + CheckResult( + name=f"metric.{category}.{metric_name}", + category="metric", + passed=False, + message=f"{metric_name}: query failed ({exc})", + ) + ) + + +# --------------------------------------------------------------------------- +# Log-Trace Correlation Validation (Loki API) +# --------------------------------------------------------------------------- + + +async def validate_log_trace_correlation( + session: aiohttp.ClientSession, + loki_url: str, + tempo_url: str, + report: ValidationReport, +) -> None: + """Validate that Loki logs contain trace_id/span_id for correlation. + + Checks: + 1. Logs with trace_id= field exist in Loki. + 2. A random trace_id from Tempo can be found in Loki logs. + + Args: + session: aiohttp client session. + loki_url: Base URL for Loki API (e.g., http://localhost:3100). + tempo_url: Base URL for Tempo API. + report: ValidationReport to accumulate results. + """ + logger.info("--- Log-Trace Correlation Validation (Loki) ---") + + # Check 1: Any logs with trace_id exist. + try: + params = { + "query": '{job="xrpld"} |= "trace_id="', + "limit": 5, + "direction": "backward", + } + async with session.get( + f"{loki_url}/loki/api/v1/query_range", params=params + ) as resp: + data = await resp.json() + streams = data.get("data", {}).get("result", []) + total_entries = sum(len(s.get("values", [])) for s in streams) + report.add( + CheckResult( + name="log.trace_id_present", + category="log", + passed=total_entries > 0, + message=( + f"Found {total_entries} log entries with trace_id" + if total_entries > 0 + else "No log entries with trace_id found" + ), + details={"log_count": total_entries}, + ) + ) + except Exception as exc: + report.add( + CheckResult( + name="log.trace_id_present", + category="log", + passed=False, + message=f"Loki query failed: {exc}", + ) + ) + + # Check 2: Cross-reference a trace_id from Tempo to Loki. + try: + # Get a recent trace from Tempo. + traces = await _tempo_search( + session, + tempo_url, + '{resource.service.name="xrpld"}', + limit=1, + ) + + if traces: + trace_id = traces[0].get("traceID", "") + if trace_id: + # Search Loki for this trace_id. + loki_params = { + "query": f'{{job="xrpld"}} |= "{trace_id}"', + "limit": 5, + "direction": "backward", + } + async with session.get( + f"{loki_url}/loki/api/v1/query_range", + params=loki_params, + ) as loki_resp: + loki_data = await loki_resp.json() + loki_streams = loki_data.get("data", {}).get("result", []) + loki_count = sum(len(s.get("values", [])) for s in loki_streams) + report.add( + CheckResult( + name="log.trace_id_cross_reference", + category="log", + passed=loki_count > 0, + message=( + f"trace_id {trace_id[:16]}... found in " + f"{loki_count} Loki entries" + if loki_count > 0 + else f"trace_id {trace_id[:16]}... not found " "in Loki" + ), + details={ + "trace_id": trace_id, + "loki_count": loki_count, + }, + ) + ) + else: + report.add( + CheckResult( + name="log.trace_id_cross_reference", + category="log", + passed=False, + message="No traces in Tempo to cross-reference", + ) + ) + except Exception as exc: + report.add( + CheckResult( + name="log.trace_id_cross_reference", + category="log", + passed=False, + message=f"Cross-reference check failed: {exc}", + ) + ) + + +# --------------------------------------------------------------------------- +# Dashboard Validation (Grafana API) +# --------------------------------------------------------------------------- + + +async def validate_dashboards( + session: aiohttp.ClientSession, + grafana_url: str, + report: ValidationReport, +) -> None: + """Validate that all Grafana dashboards are accessible and return data. + + For each expected dashboard UID, queries the Grafana API to verify + the dashboard exists and is loadable. + + Args: + session: aiohttp client session. + grafana_url: Base URL for Grafana API (e.g., http://localhost:3000). + report: ValidationReport to accumulate results. + """ + logger.info("--- Dashboard Validation (Grafana) ---") + + with open(EXPECTED_METRICS_FILE) as f: + expected = json.load(f) + + dashboard_uids = expected.get("grafana_dashboards", {}).get("uids", []) + + for uid in dashboard_uids: + try: + async with session.get(f"{grafana_url}/api/dashboards/uid/{uid}") as resp: + if resp.status == 200: + data = await resp.json() + dashboard = data.get("dashboard", {}) + panel_count = len(dashboard.get("panels", [])) + report.add( + CheckResult( + name=f"dashboard.{uid}", + category="dashboard", + passed=True, + message=(f"{uid}: loaded ({panel_count} panels)"), + details={"panel_count": panel_count}, + ) + ) + else: + report.add( + CheckResult( + name=f"dashboard.{uid}", + category="dashboard", + passed=False, + message=f"{uid}: HTTP {resp.status}", + ) + ) + except Exception as exc: + report.add( + CheckResult( + name=f"dashboard.{uid}", + category="dashboard", + passed=False, + message=f"{uid}: query failed ({exc})", + ) + ) + + +# --------------------------------------------------------------------------- +# Span duration validation +# --------------------------------------------------------------------------- + + +async def validate_span_durations( + session: aiohttp.ClientSession, + tempo_url: str, + report: ValidationReport, +) -> None: + """Validate that span durations are within reasonable bounds. + + Checks that spans have duration > 0 and < 60s, flagging any anomalies. + + Args: + session: aiohttp client session. + tempo_url: Base URL for Tempo API. + report: ValidationReport to accumulate results. + """ + logger.info("--- Span Duration Validation ---") + + try: + traces = await _tempo_search( + session, + tempo_url, + '{resource.service.name="xrpld"}', + limit=5, + ) + + if not traces: + report.add( + CheckResult( + name="span.duration_bounds", + category="span", + passed=False, + message="No traces available for duration check", + ) + ) + return + + total_spans = 0 + invalid_spans = 0 + max_duration_ns = 0 + + for trace_summary in traces: + trace_id = trace_summary.get("traceID", "") + if not trace_id: + continue + spans = await _tempo_get_trace(session, tempo_url, trace_id) + for span in spans: + start_ns = int(span.get("startTimeUnixNano", "0")) + end_ns = int(span.get("endTimeUnixNano", "0")) + duration_ns = end_ns - start_ns + total_spans += 1 + max_duration_ns = max(max_duration_ns, duration_ns) + # Invalid if negative or > 60 seconds. + if duration_ns < 0 or duration_ns > 60_000_000_000: + invalid_spans += 1 + + max_duration_ms = max_duration_ns / 1_000_000 + + report.add( + CheckResult( + name="span.duration_bounds", + category="span", + passed=invalid_spans == 0, + message=( + f"All {total_spans} spans have valid durations " + f"(max: {max_duration_ms:.1f}ms)" + if invalid_spans == 0 + else f"{invalid_spans}/{total_spans} spans have invalid " + "durations (<0 or >60s)" + ), + details={ + "total_spans": total_spans, + "invalid_spans": invalid_spans, + "max_duration_ms": round(max_duration_ms, 2), + }, + ) + ) + except Exception as exc: + report.add( + CheckResult( + name="span.duration_bounds", + category="span", + passed=False, + message=f"Duration check failed: {exc}", + ) + ) + + +# --------------------------------------------------------------------------- +# External Dashboard Parity Validation +# --------------------------------------------------------------------------- + +# Span attributes that external dashboards (validator-health, peer-quality, +# node-health) depend on. Each entry maps a span name to the +# attributes that must be present for external dashboard panels to render. +# Keys follow the 2026-05-13 span-attr naming redesign (bare/underscore form; +# dotted xrpl.* reserved for resource attributes). The amendment_blocked, +# server_state, and proposers_validated values that earlier external-dashboard +# work tracked are NOT span attributes — they exist only as MetricsRegistry +# metrics (validator_health{metric="amendment_blocked"}, +# state_tracking{metric="state_value"}, etc.), so they are validated by +# PARITY_VALUE_SANITY below rather than as span attributes here. +PARITY_SPAN_ATTRS: list[dict[str, str]] = [ + {"span": "tx.receive", "attr": "peer_version"}, + {"span": "consensus.validation.send", "attr": "ledger_hash"}, + {"span": "consensus.validation.send", "attr": "full_validation"}, + # peer.validation.receive shares the ledger_hash / full_validation keys with + # consensus.validation.send (same keys, told apart by span name). + {"span": "peer.validation.receive", "attr": "ledger_hash"}, + {"span": "peer.validation.receive", "attr": "full_validation"}, + {"span": "consensus.accept", "attr": "quorum"}, +] + +# Value sanity bounds for external-parity metrics. Each entry specifies a +# Prometheus query and the acceptable range [lo, hi] for the returned value. +PARITY_VALUE_SANITY: list[dict[str, Any]] = [ + { + "name": "validation_agreement_pct_1h", + "query": 'validation_agreement{metric="agreement_pct_1h"}', + "lo": 0, + "hi": 100, + }, + { + "name": "unl_expiry_days", + "query": 'validator_health{metric="unl_expiry_days"}', + "lo": 0, + "hi": None, + "exclusive_lo": True, + }, + { + "name": "peer_latency_p90_ms", + "query": 'peer_quality{metric="peer_latency_p90_ms"}', + "lo": 0, + "hi": None, + }, + { + "name": "state_value", + "query": 'state_tracking{metric="state_value"}', + "lo": 0, + "hi": 7, + }, +] + + +async def validate_parity_span_attrs( + session: aiohttp.ClientSession, + tempo_url: str, + report: ValidationReport, +) -> None: + """Validate span attributes required by external dashboard panels. + + For each (span, attribute) pair in PARITY_SPAN_ATTRS, queries Tempo + for the span and checks that the attribute key exists on at least one + span in the returned traces. + + Args: + session: aiohttp client session. + tempo_url: Base URL for Tempo API. + report: ValidationReport to accumulate results. + """ + logger.info("--- External Parity: Span Attribute Checks ---") + + for entry in PARITY_SPAN_ATTRS: + span_name = entry["span"] + attr_name = entry["attr"] + check_name = f"parity.span_attr.{span_name}.{attr_name}" + + try: + query = '{resource.service.name="xrpld" && name="' + span_name + '"}' + traces = await _tempo_search(session, tempo_url, query, limit=5) + + if not traces: + report.add( + CheckResult( + name=check_name, + category="parity", + passed=False, + message=( + f"{span_name}: no traces found, " + f"cannot verify attr {attr_name}" + ), + ) + ) + continue + + # Fetch full trace and search spans for the attribute. + found = False + for trace_summary in traces: + trace_id = trace_summary.get("traceID", "") + if not trace_id: + continue + spans = await _tempo_get_trace(session, tempo_url, trace_id) + for span in spans: + if attr_name in _otlp_span_attr_keys(span): + found = True + break + if found: + break + + report.add( + CheckResult( + name=check_name, + category="parity", + passed=found, + message=( + f"{span_name}: attribute '{attr_name}' present" + if found + else f"{span_name}: attribute '{attr_name}' missing" + ), + ) + ) + except Exception as exc: + report.add( + CheckResult( + name=check_name, + category="parity", + passed=False, + message=f"{span_name}: attr check failed ({exc})", + ) + ) + + +async def validate_parity_value_sanity( + session: aiohttp.ClientSession, + prometheus_url: str, + report: ValidationReport, +) -> None: + """Validate that external-parity metric values fall within sane bounds. + + For each entry in PARITY_VALUE_SANITY, queries the current value from + Prometheus and checks it against the specified [lo, hi] range. + + Args: + session: aiohttp client session. + prometheus_url: Prometheus API base URL. + report: ValidationReport to accumulate results. + """ + logger.info("--- External Parity: Value Sanity Checks ---") + + for entry in PARITY_VALUE_SANITY: + name = entry["name"] + query = entry["query"] + lo = entry["lo"] + hi = entry["hi"] + exclusive_lo = entry.get("exclusive_lo", False) + check_name = f"parity.value_sanity.{name}" + + try: + params = {"query": query} + async with session.get( + f"{prometheus_url}/api/v1/query", params=params + ) as resp: + data = await resp.json() + results = data.get("data", {}).get("result", []) + + if not results: + report.add( + CheckResult( + name=check_name, + category="parity", + passed=False, + message=f"{name}: no data returned from Prometheus", + ) + ) + continue + + # Use the first result's value. + value = float(results[0]["value"][1]) + + # Check bounds. + in_range = True + if exclusive_lo: + in_range = in_range and (value > lo) + else: + in_range = in_range and (value >= lo) + if hi is not None: + in_range = in_range and (value <= hi) + + # Build human-readable bound description. + lo_op = ">" if exclusive_lo else ">=" + bound_desc = f"{lo_op} {lo}" + if hi is not None: + bound_desc += f" and <= {hi}" + + report.add( + CheckResult( + name=check_name, + category="parity", + passed=in_range, + message=( + f"{name}: value {value} is within bounds ({bound_desc})" + if in_range + else f"{name}: value {value} out of bounds " + f"(expected {bound_desc})" + ), + details={"value": value, "lo": lo, "hi": hi}, + ) + ) + except Exception as exc: + report.add( + CheckResult( + name=check_name, + category="parity", + passed=False, + message=f"{name}: sanity check failed ({exc})", + ) + ) + + +# --------------------------------------------------------------------------- +# Main validation orchestrator +# --------------------------------------------------------------------------- + + +async def run_validation( + tempo_url: str, + prometheus_url: str, + loki_url: str, + grafana_url: str, + skip_loki: bool = False, +) -> ValidationReport: + """Run all validation checks and return a report. + + Args: + tempo_url: Tempo API base URL. + prometheus_url: Prometheus API base URL. + loki_url: Loki API base URL. + grafana_url: Grafana API base URL. + skip_loki: If True, skip log-trace correlation checks. + + Returns: + ValidationReport with all check results. + """ + report = ValidationReport() + report.start_time = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + async with aiohttp.ClientSession() as session: + await validate_spans(session, tempo_url, report) + await validate_span_durations(session, tempo_url, report) + await validate_metrics(session, prometheus_url, report) + if not skip_loki: + await validate_log_trace_correlation(session, loki_url, tempo_url, report) + await validate_dashboards(session, grafana_url, report) + await validate_parity_span_attrs(session, tempo_url, report) + await validate_parity_value_sanity(session, prometheus_url, report) + + report.end_time = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + return report + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Telemetry Validation Suite for xrpld", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Run all validations with defaults: + python3 validate_telemetry.py + + # Write report to file: + python3 validate_telemetry.py --report /tmp/validation-report.json + + # Custom endpoints: + python3 validate_telemetry.py \\ + --tempo http://tempo:3200 --prometheus http://prom:9090 + + # Skip Loki checks (if log-trace correlation is not set up): + python3 validate_telemetry.py --skip-loki + """, + ) + parser.add_argument( + "--tempo", + type=str, + default=DEFAULT_TEMPO, + help=f"Tempo API URL (default: {DEFAULT_TEMPO})", + ) + parser.add_argument( + "--prometheus", + type=str, + default=DEFAULT_PROMETHEUS, + help=f"Prometheus API URL (default: {DEFAULT_PROMETHEUS})", + ) + parser.add_argument( + "--loki", + type=str, + default=DEFAULT_LOKI, + help=f"Loki API URL (default: {DEFAULT_LOKI})", + ) + parser.add_argument( + "--grafana", + type=str, + default=DEFAULT_GRAFANA, + help=f"Grafana API URL (default: {DEFAULT_GRAFANA})", + ) + parser.add_argument( + "--skip-loki", + action="store_true", + help="Skip log-trace correlation validation", + ) + parser.add_argument( + "--report", + type=str, + default=None, + help="Write JSON report to this file path", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable debug logging", + ) + return parser.parse_args() + + +def main() -> None: + """Main entry point for the telemetry validation suite.""" + args = parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s %(message)s", + ) + + report = asyncio.run( + run_validation( + tempo_url=args.tempo, + prometheus_url=args.prometheus, + loki_url=args.loki, + grafana_url=args.grafana, + skip_loki=args.skip_loki, + ) + ) + + # Print summary. + print("") + print("=" * 60) + print(" TELEMETRY VALIDATION REPORT") + print("=" * 60) + print(f" Total checks: {report.total_checks}") + print(f" Passed: {report.passed}") + print(f" Failed: {report.failed}") + print("=" * 60) + print("") + + # Print failures. + if report.failed > 0: + print("FAILED CHECKS:") + for check in report.checks: + if not check.passed: + print(f" [{check.category}] {check.name}: {check.message}") + print("") + + # Write report file. + report_dict = report.to_dict() + if args.report: + with open(args.report, "w") as f: + json.dump(report_dict, f, indent=2) + logger.info("Report written to %s", args.report) + else: + print(json.dumps(report_dict, indent=2)) + + # Exit with appropriate code for CI. + sys.exit(0 if report.all_passed else 1) + + +if __name__ == "__main__": + main() diff --git a/docker/telemetry/workload/workload-profiles.json b/docker/telemetry/workload/workload-profiles.json new file mode 100644 index 00000000000..84abcf3ed00 --- /dev/null +++ b/docker/telemetry/workload/workload-profiles.json @@ -0,0 +1,108 @@ +{ + "profiles": { + "full-validation": { + "description": "Full 18-dashboard coverage with burst/idle/plateau patterns", + "phases": [ + { + "name": "warmup", + "description": "Low load to populate baseline gauges and node health metrics", + "duration_sec": 30, + "rpc": { + "rate": 5, + "weights": { "server_info": 50, "fee": 30, "ledger": 20 } + }, + "tx": null + }, + { + "name": "steady-state", + "description": "Medium sustained load — plateau data for all dashboards", + "duration_sec": 60, + "rpc": { "rate": 30 }, + "tx": { "tps": 3 } + }, + { + "name": "rpc-burst", + "description": "Heavy RPC to saturate job queue and spike latency", + "duration_sec": 30, + "rpc": { "rate": 100 }, + "tx": null + }, + { + "name": "tx-flood", + "description": "High TX rate for fee escalation and TxQ pressure", + "duration_sec": 30, + "rpc": { "rate": 5, "weights": { "server_info": 50, "fee": 50 } }, + "tx": { + "tps": 20, + "weights": { "Payment": 70, "OfferCreate": 20, "TrustSet": 10 } + } + }, + { + "name": "txq-burst", + "description": "Single-type Payment burst at high TPS to force open-ledger fee escalation and TxQ queueing, exercising the txq.* spans (txq.enqueue / txq.accept / txq.accept_tx / txq.cleanup)", + "duration_sec": 30, + "rpc": { "rate": 5, "weights": { "fee": 100 } }, + "tx": { + "tps": 60, + "weights": { "Payment": 100 } + } + }, + { + "name": "mixed-peak", + "description": "Realistic peak load — consensus and ledger ops under stress", + "duration_sec": 60, + "rpc": { "rate": 50 }, + "tx": { "tps": 10 } + }, + { + "name": "cooldown", + "description": "Low load for recovery metrics and state transition data", + "duration_sec": 30, + "rpc": { "rate": 5, "weights": { "server_info": 80, "fee": 20 } }, + "tx": null + } + ], + "propagation_wait_sec": 60 + }, + "quick-smoke": { + "description": "Fast smoke test — minimal data for CI quick checks", + "phases": [ + { + "name": "smoke", + "description": "Single phase covering all generator types", + "duration_sec": 30, + "rpc": { "rate": 20 }, + "tx": { "tps": 3 } + } + ], + "propagation_wait_sec": 30 + }, + "stress": { + "description": "Heavy sustained load for performance benchmarking", + "phases": [ + { + "name": "ramp-up", + "description": "Gradually increasing load", + "duration_sec": 30, + "rpc": { "rate": 20 }, + "tx": { "tps": 5 } + }, + { + "name": "peak", + "description": "Maximum sustained load", + "duration_sec": 120, + "rpc": { "rate": 150 }, + "tx": { "tps": 25 } + }, + { + "name": "sustain", + "description": "Continued high load for stability check", + "duration_sec": 60, + "rpc": { "rate": 100 }, + "tx": { "tps": 15 } + } + ], + "propagation_wait_sec": 60 + } + } +} diff --git a/docker/telemetry/workload/workload_orchestrator.py b/docker/telemetry/workload/workload_orchestrator.py new file mode 100755 index 00000000000..d9697d49434 --- /dev/null +++ b/docker/telemetry/workload/workload_orchestrator.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python3 +"""Workload Orchestrator for rippled telemetry validation. + +Reads a named profile from workload-profiles.json and executes sequential +load phases, each with configurable RPC and TX parameters. Produces a +combined report with per-phase results. + +Phases run sequentially. Within each phase, the RPC load generator and +transaction submitter run concurrently (if both are configured). + +Orchestration Flow:: + + workload-profiles.json + | + v + workload_orchestrator.py + | + +----+----+----+----+----+----+ + | Phase 1 | Phase 2 | ...... | Phase N | + +----+----+----+----+----+----+ + | | + +----+----+ +----+----+ + | rpc_load | | tx_sub | (concurrent within phase) + | _gen.py | | mitter | + +----+----+ +----+----+ + | | + v v + per-phase JSON reports + | + v + combined-report.json + +Usage: + python3 workload_orchestrator.py --profile full-validation + python3 workload_orchestrator.py --profile quick-smoke --endpoints ws://localhost:6006 + python3 workload_orchestrator.py --profile stress --report /tmp/report.json + +Profiles are defined in workload-profiles.json in the same directory. +""" + +import argparse +import asyncio +import json +import logging +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger("workload_orchestrator") + +SCRIPT_DIR = Path(__file__).parent.resolve() +PROFILES_FILE = SCRIPT_DIR / "workload-profiles.json" + + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclass +class PhaseResult: + """Result of a single workload phase. + + Attributes: + name: Phase name from the profile. + duration_sec: Configured duration. + actual_sec: Actual elapsed time. + rpc_summary: JSON summary from rpc_load_generator, or None. + tx_summary: JSON summary from tx_submitter, or None. + errors: List of error messages from subprocess failures. + """ + + name: str + duration_sec: int + actual_sec: float = 0.0 + rpc_summary: dict[str, Any] | None = None + tx_summary: dict[str, Any] | None = None + errors: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Profile loading +# --------------------------------------------------------------------------- + + +def load_profile(profile_name: str) -> dict[str, Any]: + """Load a named profile from workload-profiles.json. + + Args: + profile_name: Key in the profiles dict (e.g., "full-validation"). + + Returns: + The profile dict with phases and propagation_wait_sec. + + Raises: + SystemExit: If the profile file or name is not found. + """ + if not PROFILES_FILE.exists(): + logger.error("Profiles file not found: %s", PROFILES_FILE) + sys.exit(2) + + with open(PROFILES_FILE) as f: + data = json.load(f) + + profiles = data.get("profiles", {}) + if profile_name not in profiles: + available = ", ".join(profiles.keys()) + logger.error("Profile '%s' not found. Available: %s", profile_name, available) + sys.exit(2) + + profile = profiles[profile_name] + + # Validate profile schema — fail fast on bad config. + phases = profile.get("phases", []) + if not isinstance(phases, list) or not phases: + logger.error("Profile '%s' has no valid phases", profile_name) + sys.exit(2) + for i, phase in enumerate(phases): + if not isinstance(phase.get("name"), str): + logger.error("Phase %d missing valid 'name'", i) + sys.exit(2) + if ( + not isinstance(phase.get("duration_sec"), (int, float)) + or phase["duration_sec"] <= 0 + ): + logger.error( + "Phase %d '%s' has invalid duration_sec", + i, + phase.get("name"), + ) + sys.exit(2) + + logger.info( + "Loaded profile '%s': %s (%d phases)", + profile_name, + profile.get("description", ""), + len(phases), + ) + return profile + + +# --------------------------------------------------------------------------- +# Subprocess execution +# --------------------------------------------------------------------------- + + +async def run_subprocess(cmd: list[str], label: str) -> tuple[int, str, str]: + """Run a subprocess and capture its stdout and stderr. + + Args: + cmd: Command and arguments. + label: Human-readable label for logging. + + Returns: + Tuple of (return_code, stdout_text, stderr_text). + """ + logger.debug("Starting %s: %s", label, " ".join(cmd)) + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + logger.warning( + "%s exited with code %d: %s", + label, + proc.returncode, + stderr.decode().strip()[-500:], + ) + return proc.returncode, stdout.decode(), stderr.decode() + + +# --------------------------------------------------------------------------- +# Phase execution +# --------------------------------------------------------------------------- + + +def _collect_task_result( + label: str, + returncode: int, + stderr: str, + report_path: Path, + result: PhaseResult, +) -> None: + """Process the result of a completed subprocess task. + + Reads the JSON report file (if it exists) and records any errors. + + Args: + label: "rpc" or "tx". + returncode: Subprocess exit code. + stderr: Captured stderr text. + report_path: Path to the JSON report file. + result: PhaseResult to update. + """ + if report_path.exists(): + try: + with open(report_path) as f: + summary = json.load(f) + if label == "rpc": + result.rpc_summary = summary + elif label == "tx": + result.tx_summary = summary + except (json.JSONDecodeError, OSError) as exc: + logger.warning("Failed to parse %s report %s: %s", label, report_path, exc) + result.errors.append(f"Failed to parse {label} report: {exc}") + + if returncode != 0: + snippet = stderr.strip()[-200:] if stderr else "" + result.errors.append( + f"{label.upper()} generator exited with {returncode}: {snippet}" + ) + + +def _build_rpc_cmd( + endpoints: list[str], rpc_cfg: dict[str, Any], duration: int, output: Path +) -> list[str]: + """Build the command list for the RPC load generator subprocess.""" + cmd = [ + sys.executable, + str(SCRIPT_DIR / "rpc_load_generator.py"), + "--endpoints", + *endpoints, + "--rate", + str(rpc_cfg.get("rate", 50)), + "--duration", + str(duration), + "--output", + str(output), + ] + weights = rpc_cfg.get("weights") + if weights: + cmd.extend(["--weights", json.dumps(weights)]) + return cmd + + +def _build_tx_cmd( + endpoint: str, tx_cfg: dict[str, Any], duration: int, output: Path +) -> list[str]: + """Build the command list for the TX submitter subprocess.""" + cmd = [ + sys.executable, + str(SCRIPT_DIR / "tx_submitter.py"), + "--endpoint", + endpoint, + "--tps", + str(tx_cfg.get("tps", 5)), + "--duration", + str(duration), + "--output", + str(output), + ] + weights = tx_cfg.get("weights") + if weights: + cmd.extend(["--weights", json.dumps(weights)]) + return cmd + + +async def run_phase( + phase: dict[str, Any], + endpoints: list[str], + report_dir: Path, + phase_idx: int, +) -> PhaseResult: + """Execute a single workload phase. + + Launches rpc_load_generator.py and/or tx_submitter.py as subprocesses + based on the phase configuration. Both run concurrently if configured. + + Args: + phase: Phase dict from the profile. + endpoints: List of WebSocket endpoint URLs. + report_dir: Directory for per-phase JSON reports. + phase_idx: Phase index (for file naming). + + Returns: + PhaseResult with subprocess outputs. + """ + name = phase["name"] + duration = phase["duration_sec"] + result = PhaseResult(name=name, duration_sec=duration) + prefix = f"phase{phase_idx + 1}-{name}" + + logger.info( + "=== Phase %d: %s (%ds) — %s ===", + phase_idx + 1, + name, + duration, + phase.get("description", ""), + ) + + tasks: list[tuple[str, Path, asyncio.Task]] = [] + t0 = time.monotonic() + + rpc_cfg = phase.get("rpc") + if rpc_cfg: + rpc_out = report_dir / f"{prefix}-rpc.json" + cmd = _build_rpc_cmd(endpoints, rpc_cfg, duration, rpc_out) + tasks.append( + ("rpc", rpc_out, asyncio.create_task(run_subprocess(cmd, f"RPC [{name}]"))) + ) + + tx_cfg = phase.get("tx") + if tx_cfg: + tx_out = report_dir / f"{prefix}-tx.json" + cmd = _build_tx_cmd(endpoints[0], tx_cfg, duration, tx_out) + tasks.append( + ("tx", tx_out, asyncio.create_task(run_subprocess(cmd, f"TX [{name}]"))) + ) + + if not tasks: + logger.warning( + "Phase %d: %s — no workload configured, skipping", phase_idx + 1, name + ) + return result + + for label, report_path, task in tasks: + returncode, _stdout, stderr = await task + _collect_task_result(label, returncode, stderr, report_path, result) + + result.actual_sec = time.monotonic() - t0 + logger.info( + "Phase %d complete: %.1fs actual, %d errors", + phase_idx + 1, + result.actual_sec, + len(result.errors), + ) + return result + + +# --------------------------------------------------------------------------- +# Profile execution +# --------------------------------------------------------------------------- + + +async def run_profile( + profile: dict[str, Any], + endpoints: list[str], + report_dir: Path, +) -> dict[str, Any]: + """Execute all phases in a profile sequentially. + + Args: + profile: Profile dict with phases and propagation_wait_sec. + endpoints: WebSocket endpoints for the rippled cluster. + report_dir: Directory for phase reports. + + Returns: + Combined report dict with per-phase results and totals. + """ + phases = profile.get("phases", []) + propagation_wait = profile.get("propagation_wait_sec", 60) + results: list[PhaseResult] = [] + + total_start = time.monotonic() + + for idx, phase in enumerate(phases): + result = await run_phase(phase, endpoints, report_dir, idx) + results.append(result) + + # Wait for telemetry data to propagate through the collector pipeline. + logger.info("Waiting %ds for telemetry data to propagate...", propagation_wait) + await asyncio.sleep(propagation_wait) + + total_elapsed = time.monotonic() - total_start + + # Build combined report from all phase results. + total_rpc_sent = 0 + total_rpc_errors = 0 + total_tx_submitted = 0 + total_tx_errors = 0 + phase_reports = [] + + for r in results: + pr: dict[str, Any] = { + "name": r.name, + "duration_sec": r.duration_sec, + "actual_sec": round(r.actual_sec, 1), + "errors": r.errors, + } + if r.rpc_summary: + pr["rpc"] = r.rpc_summary + total_rpc_sent += r.rpc_summary.get("total_sent", 0) + total_rpc_errors += r.rpc_summary.get("total_errors", 0) + if r.tx_summary: + pr["tx"] = r.tx_summary + total_tx_submitted += r.tx_summary.get("total_submitted", 0) + total_tx_errors += r.tx_summary.get("total_errors", 0) + phase_reports.append(pr) + + report = { + "profile": profile.get("description", ""), + "total_elapsed_sec": round(total_elapsed, 1), + "phases": phase_reports, + "totals": { + "rpc_sent": total_rpc_sent, + "rpc_errors": total_rpc_errors, + "tx_submitted": total_tx_submitted, + "tx_errors": total_tx_errors, + }, + } + + return report + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Workload Orchestrator for rippled telemetry validation", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Profiles: + full-validation Full 18-dashboard coverage (~5 min load + 1 min propagation) + quick-smoke Fast CI smoke test (~30s load + 30s propagation) + stress Heavy sustained load for benchmarking (~3.5 min + 1 min) + +Examples: + python3 workload_orchestrator.py --profile full-validation + python3 workload_orchestrator.py --profile quick-smoke --endpoints ws://localhost:6006 + python3 workload_orchestrator.py --profile stress --report /tmp/report.json + """, + ) + parser.add_argument( + "--profile", + type=str, + required=True, + help="Named profile from workload-profiles.json", + ) + parser.add_argument( + "--endpoints", + nargs="+", + default=["ws://localhost:6006"], + help="WebSocket endpoints (default: ws://localhost:6006)", + ) + parser.add_argument( + "--report", + type=str, + default=None, + help="Write combined JSON report to this file", + ) + parser.add_argument( + "--report-dir", + type=str, + default="/tmp/xrpld-validation/reports", + help="Directory for per-phase reports", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable debug logging", + ) + return parser.parse_args() + + +def main() -> None: + """Main entry point for the workload orchestrator.""" + args = parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s %(message)s", + ) + + profile = load_profile(args.profile) + report_dir = Path(args.report_dir) + report_dir.mkdir(parents=True, exist_ok=True) + + report = asyncio.run(run_profile(profile, args.endpoints, report_dir)) + + print(json.dumps(report, indent=2)) + + if args.report: + with open(args.report, "w") as f: + json.dump(report, f, indent=2) + logger.info("Combined report written to %s", args.report) + + # Exit with error if either generator had high error rates. + totals = report["totals"] + rpc_err_rate = ( + totals["rpc_errors"] / totals["rpc_sent"] * 100 if totals["rpc_sent"] > 0 else 0 + ) + tx_err_rate = ( + totals["tx_errors"] / totals["tx_submitted"] * 100 + if totals["tx_submitted"] > 0 + else 0 + ) + # TX threshold is higher because short-lived CI test environments lack + # pre-funded accounts, causing expected failures for complex transactions + # (AMMCreate, EscrowFinish, etc.) that require specific ledger state. + if rpc_err_rate > 50 or tx_err_rate > 95: + logger.error( + "High error rates: RPC=%.1f%%, TX=%.1f%%", rpc_err_rate, tx_err_rate + ) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/docker/telemetry/workload/xrpld-validator.cfg.template b/docker/telemetry/workload/xrpld-validator.cfg.template new file mode 100644 index 00000000000..725faf550cf --- /dev/null +++ b/docker/telemetry/workload/xrpld-validator.cfg.template @@ -0,0 +1,96 @@ +# xrpld validator node configuration template for workload harness. +# +# Placeholders (replaced by docker-compose entrypoint): +# {{NODE_INDEX}} — Node number (1-based) +# {{RPC_PORT}} — HTTP RPC port +# {{WS_PORT}} — WebSocket port +# {{PEER_PORT}} — Peer protocol port +# {{DATA_DIR}} — Node data directory +# {{VALIDATION_SEED}} — Validator seed from key generation +# {{VALIDATORS_FILE}} — Path to shared validators.txt +# {{IPS_FIXED}} — Peer addresses (one per line) +# {{OTEL_ENDPOINT}} — OTel Collector OTLP/HTTP traces endpoint +# {{OTEL_METRICS_ENDPOINT}} — OTel Collector OTLP/HTTP metrics endpoint +# {{LOG_LEVEL}} — Log level (debug, info, warning, error) + +[server] +port_rpc +port_ws +port_peer + +[port_rpc] +port = {{RPC_PORT}} +ip = 0.0.0.0 +admin = 0.0.0.0 +protocol = http + +[port_ws] +port = {{WS_PORT}} +ip = 0.0.0.0 +admin = 0.0.0.0 +protocol = ws + +[port_peer] +port = {{PEER_PORT}} +ip = 0.0.0.0 +protocol = peer + +[node_db] +type=NuDB +path={{DATA_DIR}}/nudb +online_delete=256 + +[database_path] +{{DATA_DIR}}/db + +[debug_logfile] +{{DATA_DIR}}/debug.log + +[validation_seed] +{{VALIDATION_SEED}} + +[validators_file] +{{VALIDATORS_FILE}} + +[ips_fixed] +{{IPS_FIXED}} + +[peer_private] +1 + +# --- OpenTelemetry tracing (all categories enabled) --- +[telemetry] +enabled=1 +service_instance_id=validator-{{NODE_INDEX}} +endpoint={{OTEL_ENDPOINT}} +exporter=otlp_http +batch_size=512 +batch_delay_ms=2000 +max_queue_size=2048 +trace_rpc=1 +trace_transactions=1 +trace_consensus=1 +trace_peer=1 +trace_ledger=1 + +# --- Native OTel metrics (beast::insight over OTLP/HTTP) --- +# The collector has no StatsD receiver (metrics pipeline is [otlp, spanmetrics]), +# so beast::insight exports natively over OTLP. prefix=xrpld matches the OTel +# resource service name and the metric names the dashboards query. +[insight] +server=otel +endpoint={{OTEL_METRICS_ENDPOINT}} +prefix=xrpld + +[rpc_startup] +{ "command": "log_level", "severity": "{{LOG_LEVEL}}" } + +[ssl_verify] +0 + +# --- Network tuning for local cluster --- +[network_id] +0 + +[sntp_servers] +time.google.com diff --git a/docker/telemetry/xrpld-telemetry-mainnet.cfg b/docker/telemetry/xrpld-telemetry-mainnet.cfg new file mode 100644 index 00000000000..afca57e4fa0 --- /dev/null +++ b/docker/telemetry/xrpld-telemetry-mainnet.cfg @@ -0,0 +1,141 @@ +# xrpld configuration for Mainnet with full OpenTelemetry tracing. +# +# Connects to XRP Ledger Mainnet as an observer-only tracking node +# (no validator credentials) and exercises ALL instrumented workflows: +# RPC, transactions, consensus, peer overlay, ledger ops, and +# pathfinding. +# +# Usage: +# 1. Start the observability stack: +# docker compose -f docker/telemetry/docker-compose.yml up -d +# 2. Run xrpld: +# ./xrpld --conf docker/telemetry/xrpld-telemetry.cfg +# 3. Wait for sync (server_state=full), then exercise workflows: +# curl -s http://localhost:5005 -d '{"method":"server_info"}' +# 4. View traces in Grafana Explore -> Tempo: http://localhost:3000 + +# --- Server ports ----------------------------------------------------------- + +[server] +port_rpc_admin_local +port_ws_admin_local +port_ws_public +port_peer + +[port_rpc_admin_local] +port = 5005 +ip = 127.0.0.1 +admin = 127.0.0.1 +protocol = http + +[port_ws_admin_local] +port = 6006 +ip = 127.0.0.1 +admin = 127.0.0.1 +protocol = ws + +[port_ws_public] +port = 6005 +ip = 0.0.0.0 +protocol = ws + +[port_peer] +port = 51235 +ip = 0.0.0.0 +protocol = peer + +# --- Network ---------------------------------------------------------------- + +[network_id] +main + +# [ips] omitted on purpose. xrpld uses its built-in mainnet hub list +# (r.ripple.com, sahyadri.isrdc.in, hubs.xrpkuwait.com, +# hub.xrpl-commons.org) for peer discovery. + +[validators_file] +validators-mainnet.txt + +[peer_private] +0 + +[peers_max] +21 + +# --- Pathfinding (exercises ripple_path_find / path_find workflows) --------- + +[path_search] +7 + +[path_search_fast] +2 + +[path_search_max] +10 + +# --- Signing (allows sign/sign_for RPC for test tx submission) -------------- + +[signing_support] +true + +# --- Database --------------------------------------------------------------- + +[node_db] +type=NuDB +path=docker/telemetry/data/nudb +online_delete=2000 +advisory_delete=0 + +[database_path] +docker/telemetry/data + +[ledger_history] +1000 + +# --- Logging ---------------------------------------------------------------- + +# Path is resolved relative to this config file's directory (docker/telemetry), +# so this writes to docker/telemetry/data/logs/mainnet/debug.log — the same +# dir the compose stack bind-mounts into the collector as /var/log/xrpld. +[debug_logfile] +data/logs/mainnet/debug.log + +[rpc_startup] +{ "command": "log_level", "severity": "warning" } + +# --- SSL -------------------------------------------------------------------- + +[ssl_verify] +0 + +# --- Insight (native OTel metrics via beast::insight) ----------------------- + +[insight] +server=otel +endpoint=http://localhost:4318/v1/metrics +prefix=xrpld +# Sets the OTel service.instance.id resource attribute, which Prometheus +# exposes as the `service_instance_id` label. Dashboards filter on it via the +# $node template variable, so without this every insight-backed panel is +# empty. Matches [telemetry] service_instance_id for a single node identity. +service_instance_id=xrpld-mainnet + +# --- OpenTelemetry tracing -------------------------------------------------- + +[telemetry] +enabled=1 +service_instance_id=xrpld-mainnet +endpoint=http://localhost:4318/v1/traces +metrics_endpoint=http://localhost:4318/v1/metrics +exporter=otlp_http +# Mainnet has high span throughput across peer/ledger/consensus. Head +# sampling is fixed at 1.0 (sample everything) and not configurable; +# reduce Tempo/collector load with collector-side tail sampling. +batch_size=512 +batch_delay_ms=5000 +max_queue_size=2048 +trace_rpc=1 +trace_transactions=1 +trace_consensus=1 +trace_peer=1 +trace_ledger=1 diff --git a/docker/telemetry/xrpld-telemetry.cfg b/docker/telemetry/xrpld-telemetry.cfg new file mode 100644 index 00000000000..bd7454e5983 --- /dev/null +++ b/docker/telemetry/xrpld-telemetry.cfg @@ -0,0 +1,136 @@ +# xrpld configuration for Devnet with full OpenTelemetry tracing. +# +# Connects to the XRP Ledger Devnet and exercises ALL instrumented +# workflows: RPC, transactions, consensus, peer overlay, ledger ops, +# and pathfinding. +# +# Usage: +# 1. Start the observability stack: +# docker compose -f docker/telemetry/docker-compose.yml up -d +# 2. Run xrpld: +# ./xrpld --conf docker/telemetry/xrpld-telemetry.cfg +# 3. Wait for sync (server_state=full), then exercise workflows: +# curl -s http://localhost:5005 -d '{"method":"server_info"}' +# 4. View traces in Grafana Explore -> Tempo: http://localhost:3000 + +# --- Server ports ----------------------------------------------------------- + +[server] +port_rpc_admin_local +port_ws_admin_local +port_ws_public +port_peer + +[port_rpc_admin_local] +port = 5005 +ip = 127.0.0.1 +admin = 127.0.0.1 +protocol = http + +[port_ws_admin_local] +port = 6006 +ip = 127.0.0.1 +admin = 127.0.0.1 +protocol = ws + +[port_ws_public] +port = 6005 +ip = 0.0.0.0 +protocol = ws + +[port_peer] +port = 51235 +ip = 0.0.0.0 +protocol = peer + +# --- Network ---------------------------------------------------------------- + +[network_id] +devnet + +[ips] +s.devnet.rippletest.net 51235 + +[validators_file] +validators-devnet.txt + +[peer_private] +0 + +[peers_max] +21 + +# --- Pathfinding (exercises ripple_path_find / path_find workflows) --------- + +[path_search] +7 + +[path_search_fast] +2 + +[path_search_max] +10 + +# --- Signing (allows sign/sign_for RPC for test tx submission) -------------- + +[signing_support] +true + +# --- Database --------------------------------------------------------------- + +[node_db] +type=NuDB +path=docker/telemetry/data/nudb +online_delete=2000 +advisory_delete=0 + +[database_path] +docker/telemetry/data + +[ledger_history] +1000 + +# --- Logging ---------------------------------------------------------------- + +# Path is resolved relative to this config file's directory (docker/telemetry), +# so this writes to docker/telemetry/data/logs/devnet/debug.log — the same +# dir the compose stack bind-mounts into the collector as /var/log/xrpld. +[debug_logfile] +data/logs/devnet/debug.log + +[rpc_startup] +{ "command": "log_level", "severity": "debug" } + +# --- SSL -------------------------------------------------------------------- + +[ssl_verify] +0 + +# --- Insight (native OTel metrics via beast::insight) ----------------------- + +[insight] +server=otel +endpoint=http://localhost:4318/v1/metrics +prefix=xrpld +# Sets the OTel service.instance.id resource attribute, which Prometheus +# exposes as the `service_instance_id` label. Dashboards filter on it via the +# $node template variable, so without this every insight-backed panel is +# empty. Matches [telemetry] service_instance_id for a single node identity. +service_instance_id=xrpld-devnet + +# --- OpenTelemetry tracing -------------------------------------------------- + +[telemetry] +enabled=1 +service_instance_id=xrpld-devnet +endpoint=http://localhost:4318/v1/traces +metrics_endpoint=http://localhost:4318/v1/metrics +exporter=otlp_http +batch_size=512 +batch_delay_ms=5000 +max_queue_size=2048 +trace_rpc=1 +trace_transactions=1 +trace_consensus=1 +trace_peer=1 +trace_ledger=1 diff --git a/docs/build/telemetry.md b/docs/build/telemetry.md new file mode 100644 index 00000000000..8be38466c3d --- /dev/null +++ b/docs/build/telemetry.md @@ -0,0 +1,230 @@ +# OpenTelemetry Tracing for xrpld + +This document explains how to build xrpld with OpenTelemetry distributed tracing support, configure the runtime telemetry options, and set up the observability backend to view traces. + +- [OpenTelemetry Tracing for xrpld](#opentelemetry-tracing-for-xrpld) + - [Overview](#overview) + - [Building with Telemetry](#building-with-telemetry) + - [Summary](#summary) + - [Build steps](#build-steps) + - [Install dependencies](#install-dependencies) + - [Call CMake](#call-cmake) + - [Build](#build) + - [Building without telemetry](#building-without-telemetry) + - [Troubleshooting](#troubleshooting) + - [Conan lockfile error](#conan-lockfile-error) + - [CMake target not found](#cmake-target-not-found) + - [Conditional compilation](#conditional-compilation) + - [Span lifetime and cross-thread handling](#span-lifetime-and-cross-thread-handling) + - [`SpanGuard` versus `ScopedSpanGuard`](#spanguard-versus-scopedspanguard) + - [Coroutine-aware context storage](#coroutine-aware-context-storage) + - [Handing a span to a job](#handing-a-span-to-a-job) + - [Why are unrelated spans in my trace?](#why-are-unrelated-spans-in-my-trace) + +## Overview + +xrpld supports optional [OpenTelemetry](https://opentelemetry.io/) distributed tracing. +When enabled, it instruments RPC requests with trace spans that are exported via +OTLP/HTTP to an OpenTelemetry Collector, which forwards them to a tracing backend +such as Grafana Tempo. + +Telemetry is **off by default** at both compile time and runtime: + +- **Compile time**: The Conan option `telemetry` and CMake option `telemetry` must be set to `True`/`ON`. + When disabled, all `SpanGuard` calls compile to inline no-ops (defined in `SpanGuard.h`) + with zero overhead — no OTel SDK dependency required. +- **Runtime**: The `[telemetry]` config section must set `enabled=1`. + When disabled at runtime, a no-op implementation is used. + +## Building with Telemetry + +### Summary + +Follow the same instructions as mentioned in [BUILD.md](../../BUILD.md) but with the following changes: + +1. Pass `-o telemetry=True` to `conan install` to pull the `opentelemetry-cpp` dependency. +2. CMake will automatically pick up `telemetry=ON` from the Conan-generated toolchain. +3. Build as usual. + +--- + +### Build steps + +```bash +cd /path/to/xrpld +rm -rf .build +mkdir .build +cd .build +``` + +#### Install dependencies + +The `telemetry` option adds `opentelemetry-cpp/1.26.0` as a dependency. +If the Conan lockfile does not yet include this package, bypass it with `--lockfile=""`. + +```bash +conan install .. \ + --output-folder . \ + --build missing \ + --settings build_type=Debug \ + -o telemetry=True \ + -o tests=True \ + -o xrpld=True \ + --lockfile="" +``` + +> **Note**: The first build with telemetry may take longer as `opentelemetry-cpp` +> and its transitive dependencies are compiled from source. + +#### Call CMake + +The Conan-generated toolchain file sets `telemetry=ON` automatically. +No additional CMake flags are needed beyond the standard ones. + +```bash +cmake .. -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -Dtests=ON -Dxrpld=ON +``` + +You should see in the CMake output: + +``` +-- OpenTelemetry tracing enabled +``` + +#### Build + +```bash +cmake --build . --parallel $(nproc) +``` + +## Building without telemetry + +Omit the `-o telemetry=True` option (or pass `-o telemetry=False`). +The `opentelemetry-cpp` dependency will not be downloaded, +the `XRPL_ENABLE_TELEMETRY` preprocessor define will not be set, +and all tracing macros will compile to no-ops. +The resulting binary is identical to one built before telemetry support was added. + +## Troubleshooting + +### Conan lockfile error + +If you see `ERROR: Requirement 'opentelemetry-cpp/1.26.0' not in lockfile 'requires'`, +the lockfile was generated without the telemetry dependency. +Pass `--lockfile=""` to bypass the lockfile, or regenerate it with telemetry enabled. + +### CMake target not found + +If CMake reports that `opentelemetry-cpp` targets are not found, +ensure you ran `conan install` with `-o telemetry=True` and that the +Conan-generated toolchain file is being used. +The Conan package provides a single umbrella target +`opentelemetry-cpp::opentelemetry-cpp` (not individual component targets). + +## Conditional compilation + +All OpenTelemetry SDK types are hidden behind the pimpl idiom in `SpanGuard.cpp`. +When `XRPL_ENABLE_TELEMETRY` is not defined, `SpanGuard.h` provides an all-inline +no-op stub class with zero overhead and zero OTel dependencies. +At runtime, if `enabled=0` is set in config (or the section is omitted), a +`NullTelemetry` implementation is used that returns no-op spans. +This two-layer approach ensures zero overhead when telemetry is not wanted. + +## Span lifetime and cross-thread handling + +Telemetry exposes two RAII guards with split responsibilities, plus a +non-owning activation helper. Picking the right one is what keeps a trace's +parent/child nesting and its per-line log correlation correct. + +### `SpanGuard` versus `ScopedSpanGuard` + +- **`SpanGuard`** owns a span and nothing else. It is _thread-free_: it never + touches the active-context stack, so it carries no thread affinity and may be + moved to and ended on any thread. It is movable and move-assignable. Create + one with `SpanGuard::span(cat, prefix, name)`, or with + `SpanGuard::freshRoot(...)` to start a fresh trace root that ignores whatever + span is currently ambient. Reach for a plain `SpanGuard` whenever the span + must leave the context store that created it — for example when it is handed + into a job. + +- **`ScopedSpanGuard`** owns a `SpanGuard` _plus_ an active OTel scope that + pushes the span onto the current context store. While it lives, the span is + the ambient parent for child spans created on that store, and log lines + emitted under it carry its `trace_id`. It is non-copyable and non-movable — + short-lived stack RAII. It offers the same factories (`freshRoot(...)`, + `childSpan(...)`). When the span must outlive the scope, convert it with + `operator SpanGuard() &&`: that pops the scope on the origin store and yields + the bare, thread-free `SpanGuard`. + +Rule of thumb: use `ScopedSpanGuard` for same-thread (or same-coroutine) +nesting and log correlation; use the plain `SpanGuard` whenever the span +crosses out of the store that created it. + +```mermaid +flowchart TD + SG["SpanGuard
(unscoped, thread-free)
owns span only; movable across threads and coroutines"] + SSG["ScopedSpanGuard
(scoped, store-bound)
owns a SpanGuard plus an active scope on the current store"] + SA["ScopedActivation
(non-owning)
activates a borrowed span; never owns or ends it"] + + SSG -->|"handoff: pop scope, yield bare span"| SG + SG -->|"activate / activateIfLive"| SA + SA -.->|"borrows span, no ownership"| SG + + classDef box fill:#e8f0fe,stroke:#3b5bdb,color:#111827; + class SG,SSG,SA box; +``` + +### Coroutine-aware context storage + +The active-context stack is not a plain `thread_local`. At telemetry start +xrpld installs `CoroAwareContextStorage`, which keeps the stack in an +`xrpl::LocalValue`. Because `JobQueue::Coro::resume()` swaps the coroutine's +`LocalValue` store in and out with the coroutine, the ambient context _follows +the coroutine_ across every yield and resume — even when it resumes on a +different worker thread. A `ScopedSpanGuard` held across a coroutine yield is +therefore safe: its scope rides the coroutine and pops on the same store it was +pushed onto, so it never pops the wrong stack. Off a coroutine the `LocalValue` +transparently gives each thread its own store, so behaviour matches OTel's +default thread-local storage. This is what lets the RPC entry, process, and +command spans be scoped — for correct nesting and per-line log-trace +correlation — even though the RPC path yields. + +### Handing a span to a job + +The hand-off pattern is: create a thread-free `SpanGuard` at the origin (or +convert a `ScopedSpanGuard` with `operator SpanGuard() &&`), move it into the +job closure, and inside the worker body activate it non-destructively with +`telemetry::activateIfLive(handle)`. That call takes no ownership and returns a +`ScopedActivation` (a no-op if the handle is empty or the span inactive) which +makes the span the ambient context so log lines in the worker body carry its +`trace_id`. The activation neither owns nor ends the span — the owning +`SpanGuard` still controls its lifetime and ends it when the closure is +destroyed. Keep the activation confined to a synchronous, non-yielding block. +There is no detach step: a `SpanGuard` is already thread-free. + +### Why are unrelated spans in my trace? + +Historically a scoped guard destroyed off its origin thread popped the wrong +context stack, leaving a stale ambient span in place that later work inherited. +The current design removes that failure mode in two ways: + +- **Coroutine-aware storage** makes a scope held across a coroutine yield pop on + the same store it was pushed onto, so a coroutine that resumes on another + worker never pops the wrong stack. + +- **A same-store assertion** in `ScopedSpanGuard` (and `ScopedActivation`) + records the `LocalValue` store its scope was pushed onto and checks, in + debug/test/fuzzing builds, that destruction, hand-off, and discard all happen + while that same store is active — turning a genuine cross-store misuse into an + immediate assertion failure rather than a silently corrupted trace. + +If a trace still shows unrelated spans nested under one operation, the usual +cause is an inbound entry point that inherited an ambient parent it should not +have. Start such an operation with `freshRoot()` so it begins a clean trace +root and never adopts whatever span happened to be active. To move a span +across a store boundary, keep it in a thread-free `SpanGuard` (or convert via +`operator SpanGuard() &&`) rather than holding a `ScopedSpanGuard` across the +boundary. diff --git a/docs/telemetry-glossary.md b/docs/telemetry-glossary.md new file mode 100644 index 00000000000..0bdaf6bc056 --- /dev/null +++ b/docs/telemetry-glossary.md @@ -0,0 +1,862 @@ +# Telemetry Glossary + +Plain-language definitions of the XRP Ledger and `xrpld` terms used in the +Grafana dashboard panel descriptions. Each Grafana panel already carries a short +**Keywords** gloss inline; this page is the deeper reference for going further, +and most terms link out to the canonical [xrpl.org](https://xrpl.org/docs) +documentation. + +> **Related docs**: +> [docs/telemetry-runbook.md](./telemetry-runbook.md) (operator runbook). + + + +## Contents + +- [Ledger Lifecycle](#cat-ledger-lifecycle) +- [Consensus](#cat-consensus) +- [Transaction Pipeline](#cat-transaction-pipeline) +- [Fees & Queue](#cat-fees-queue) +- [Node State & Sync](#cat-node-state-sync) +- [Peer & Overlay Networking](#cat-peer-overlay-networking) +- [Storage Internals](#cat-storage-internals) +- [Validator Health](#cat-validator-health) +- [RPC & Pathfinding](#cat-rpc-pathfinding) + + + +## Ledger Lifecycle + + + +### Ledger build + +Building a ledger means applying the agreed transaction set, in canonical order, onto the previous ledger to produce the new closed ledger and its hash. Build time is a large component of the overall close time. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Ledger build on xrpl.org](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) + + + +### Ledger close + +The XRP Ledger never converts an open ledger into a closed one; instead the server discards the open ledger and builds a new closed ledger by applying the consensus-agreed transaction set (in canonical order) on top of the previous closed ledger. Consensus triggers the close; the close completes when the new ledger is built. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Ledger close on xrpl.org](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) + + + +### Ledger close interval + +The XRP Ledger closes a new ledger at a roughly steady cadence (about every 3-5 seconds on Mainnet). Close times are rounded to a shared resolution so validators can agree on them. A node that closes far slower or faster than the network cadence is not keeping up or is misbehaving. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [Ledger close interval on xrpl.org](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) + + + +### Ledger index + +The ledger index (or ledger sequence) is the position of a ledger version in the chain, incremented by one for each new ledger. The current/open ledger index is one or two ahead of the latest validated sequence on a synced node. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [Ledger index on xrpl.org](https://xrpl.org/docs/references/protocol/data-types/basic-data-types#ledger-index) + + + +### Ledger store + +After a ledger is built and validated, the node persists it into its ledger history (the object store). The store rate should track the build rate on a healthy, in-sync node. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Ledger validation + +Validation is the stage after transaction-set agreement: each server independently computes the ledger from the agreed set, then compares results. When enough trusted validators agree on the same ledger, it is declared validated (final and immutable). A validating node issues one validation per ledger it fully validates. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Ledger validation on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) + + + +### Open ledger + +A server has exactly one open ledger: a temporary workspace where it provisionally applies transactions in the order received. Its results are tentative and can differ from the final validated result, because the closed ledger applies transactions in canonical order instead. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Open ledger on xrpl.org](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) + + + +### Published ledger + +After a ledger is validated the node publishes it to internal subscribers and clients. The published ledger normally tracks the validated ledger closely; a growing gap means the publish pipeline is backing up and subscribers may see stale data. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Transaction apply phase + +During a ledger close the server executes each transaction in the agreed set, in canonical order, updating ledger state. This apply phase (xrpld's doAccept path) is typically the largest single component of ledger-build time and scales with the number and cost of transactions in the ledger. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Transaction apply phase on xrpl.org](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) + + + +### Validated ledger + +A validated ledger is one that a quorum of trusted validators has agreed on. It is immutable and forms part of the permanent ledger history. Each ledger index has exactly one validated ledger. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [Validated ledger on xrpl.org](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) + + + +## Consensus + + + +### Clock drift + +Because each validator stamps its own observed close time, differences between validator clocks (drift) spread the proposed close times, forcing coarser close-time resolution and more distinct positions. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Clock drift on xrpl.org](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) + + + +### Close time + +Each validator proposes the wall-clock time it saw the ledger close; validators then agree on a common close time, rounded to a shared resolution so their ledgers match. Disagreement on close time forces the rounding resolution coarser and can cause extra rounds. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Close time on xrpl.org](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) + + + +### Close-time resolution + +Close times are rounded to a shared resolution (a number of seconds) so validators can agree on a single value. When validators disagree on close time the resolution moves coarser (up toward 120s); when they agree tightly it moves finer. Repeated coarsening signals persistent close-time disagreement. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Close-time resolution on xrpl.org](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) + + + +### Consensus + +XRP Ledger consensus is an iterative agreement protocol: each server listens to its trusted validators and, when a supermajority agree on the same transaction set and close time, declares consensus and builds the ledger. If they disagree, validators revise proposals over successive rounds until they converge. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Consensus on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) + + + +### Consensus mode + +Consensus mode reflects how a node is participating: Proposing (a validator advancing its own proposal), Observing (following without proposing), Wrong Ledger (working from a ledger the network disagrees with), or Switched Ledger (just changed to match the network). Sustained time in Wrong/Switched Ledger indicates the node is out of sync or flapping. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Consensus mode on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) + + + +### Consensus outcome + +The result of a consensus round. Agreed is the healthy outcome. Moved On means the node proceeded without full agreement; Expired means the round timed out; No Consensus means agreement was not reached. A growing share of non-Agreed outcomes signals network stress or connectivity loss. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Consensus outcome on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-principles-and-rules) + + + +### Consensus round + +A consensus round is one iteration in which validators relay and revise proposals. Multiple rounds (the establish count) may be needed within a single ledger before validators converge. Longer or more numerous rounds indicate disagreement, load, or poor connectivity. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Consensus round on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) + + + +### Consensus stall + +A stalled condition is flagged when consensus health checks detect that rounds are not progressing. A nonzero stall rate is an early warning that can precede ledger stalls or forks, surfacing before validated-ledger-age alarms fire. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Consensus stall on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-principles-and-rules) + + + +### Convergence time + +How long consensus took to converge on the agreed transaction set and close time, typically a few seconds. A rising convergence time indicates the network is taking longer to agree, often from load or connectivity problems. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Convergence time on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) + + + +### Distinct positions + +A count of how many different close-time positions validators held in a round. Weight on a single distinct position means everyone agreed; weight on two or more means proposals split, indicating clock drift or latency spread across the validator set. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Distinct positions on xrpl.org](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) + + + +### Establish phase + +The establish phase is the part of consensus where validators exchange and revise proposals until they converge. The establish (iteration) count per ledger is normally low (a few iterations); a growing share of ledgers needing many iterations signals disagreement or network stress. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Establish phase on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) + + + +### Fork + +A fork occurs when parts of the network validate different ledger chains. Sustained history mismatches or nodes stuck on the Wrong Ledger are fork indicators; the network is designed to avoid forks by requiring a trusted-validator quorum. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [Fork on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol) + + + +### Ledger history mismatch + +A mismatch is recorded when the ledger this node built does not hash-match the ledger the network validated. Any sustained rate indicates consensus divergence or database corruption and warrants immediate investigation; the reason label distinguishes close-time, sync-drift, and transaction-processing causes. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Position update + +Each round a node tallies disputed transactions and updates its own proposed position to move toward its trusted peers. Sustained high position-update durations point to heavy dispute resolution or slow convergence. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Position update on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) + + + +### Proposal + +During consensus each validator broadcasts a proposal: the set of candidate transactions it thinks should be in the next ledger. Validators revise proposals over rounds to match their trusted peers until a supermajority agree. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Proposal on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) + + + +### Proposers + +The number of distinct validators that proposed in the last consensus round. A falling proposer count (alongside rising convergence time) signals degrading consensus conditions, such as lost validator connectivity. + +**Scope:** network event — a network-wide consensus process; this metric is one node's view of it. + +**See also:** [Proposers on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) + + + +## Transaction Pipeline + + + +### Apply pipeline stages + +A transaction is processed in stages: preflight validates it without ledger state (signature, format), preclaim checks it against current ledger state, and apply executes it and commits state changes. A failure spike concentrated in one stage pinpoints where transactions are being rejected. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Apply pipeline stages on xrpl.org](https://xrpl.org/docs/concepts/transactions) + + + +### Direct apply vs enqueue + +When a transaction arrives, the node either applies it directly to the open ledger (if it meets the open-ledger cost) or enqueues it for a future ledger. The bypass ratio is the share applied directly; it falls as congestion pushes more transactions into the queue. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Direct apply vs enqueue on xrpl.org](https://xrpl.org/docs/concepts/transactions/transaction-queue) + + + +### Local vs relayed transactions + +Transactions either originate from a client submitting to this node (local) or are relayed from peers. Most traffic on a network node is peer-relayed; local dominates on a submission node. An unexpected surge in local submissions can indicate a client flooding the node. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Queue accept (drain) + +When a ledger closes, the node drains eligible queued transactions into it. The accept/applied ratio is the share of drained transactions that were included versus removed on failure. A healthy drain applies most of them; a low ratio means accepts are mostly failing. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Queue accept (drain) on xrpl.org](https://xrpl.org/docs/concepts/transactions/transaction-queue) + + + +### Transaction result codes + +Every transaction returns a result code grouped by prefix: tes (success), tec (failed but cost claimed, included in ledger), tef (cannot apply to this ledger or a later one), tem (malformed, cannot succeed in any ledger), ter (retry later), tel (local error). A steady background of tef/tec results is normal; a surge of one code for one type indicates a systemic issue or abusive submissions. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [Transaction result codes on xrpl.org](https://xrpl.org/docs/references/protocol/transactions/transaction-results) + + + +### Transaction suppression + +The same transaction reaches a node from many peers; the node suppresses (discards) copies it has already seen so they are not reprocessed. A large suppressed share is normal on a well-connected node; a collapse in suppression means duplicate filtering is failing. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Transaction type + +Each transaction has a type that determines its logic and cost, such as Payment, OfferCreate, TrustSet, or the AMM and NFToken families. Panels break down rate, latency, and failures by type; a single type spiking far above baseline can indicate a spam campaign of that type. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [Transaction type on xrpl.org](https://xrpl.org/docs/references/protocol/transactions/types) + + + +### Transactor + +A transactor is the code that applies a single transaction of a given type, running its type-specific checks and state changes. The transactor stage is the innermost apply step; its result is a transaction result code such as tesSUCCESS (the tes success class). + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Transactor on xrpl.org](https://xrpl.org/docs/references/protocol/transactions/transaction-results) + + + +## Fees & Queue + + + +### Base fee + +The base fee is the transaction cost a reference (cheapest) transaction must destroy under minimum load, currently 10 drops on Mainnet. The actual required fee is the base fee scaled by the load factor and, when the open ledger is busy, by fee escalation. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [Base fee on xrpl.org](https://xrpl.org/docs/concepts/transactions/transaction-cost) + + + +### drops + +A drop is the smallest denomination of XRP: 1 XRP = 1,000,000 drops. Fees, reserves, and costs are often reported in drops. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [drops on xrpl.org](https://xrpl.org/docs/references/protocol/data-types/basic-data-types#specifying-currency-amounts) + + + +### Fee escalation + +Once the open ledger holds more than its soft target number of transactions, the cost to add further transactions rises exponentially (fee escalation). Transactions that cannot pay the escalated cost are queued instead. A large gap of the open-ledger level above the reference level means escalation is active. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Fee escalation on xrpl.org](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) + + + +### Fee levels + +Fee levels express the transaction cost relative to a transaction's own minimum, so they compare across transaction types. Key levels are reference (baseline, 256), minimum (to queue), median (of the last ledger), and open-ledger (to enter the current open ledger). The open-ledger level spiking far above reference is the hallmark of congestion. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Fee levels on xrpl.org](https://xrpl.org/docs/concepts/transactions/transaction-cost#fee-levels) + + + +### In-ledger vs target count + +The node sets a soft target for how many transactions belong in a ledger, based on the previous ledger. While the in-ledger count stays at or below target, the open-ledger cost is minimal; exceeding it triggers exponential fee escalation. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [In-ledger vs target count on xrpl.org](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) + + + +### Load factor + +The load factor is a multiplier applied to the base transaction cost; 1.0 means no load. It combines local server load, network load, and cluster load. A rising factor means the node is charging premium fees due to congestion or overload; the components identify where the pressure originates. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Load factor on xrpl.org](https://xrpl.org/docs/concepts/transactions/transaction-cost#local-load-cost) + + + +### Queue admission rejection + +When the queue is at capacity or a transaction is unlikely to be included, the node refuses it entry (a drop), applying backpressure. A burst of queue_full rejections, distinct from expiry, means the node is being flooded faster than it can drain. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Queue admission rejection on xrpl.org](https://xrpl.org/docs/concepts/transactions/transaction-queue) + + + +### Queue expiry / abandonment + +A queued transaction carrying a LastLedgerSequence is dropped once that deadline passes without inclusion. A sustained expiry rate is a demand-frustration signal: submitters under-bid the escalating fee and their transactions timed out, often coinciding with spam. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Queue expiry / abandonment on xrpl.org](https://xrpl.org/docs/concepts/transactions/reliable-transaction-submission) + + + +### Reserve (base & owner) + +Reserves protect the ledger from spam by requiring accounts to hold XRP. The base reserve is the minimum per account; the owner (incremental) reserve adds a further requirement per object the account owns (offers, trust lines, escrows, etc.). Both are set by validator fee voting and reported in drops. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [Reserve (base & owner) on xrpl.org](https://xrpl.org/docs/concepts/accounts/reserves) + + + +### Transaction cost + +Every transaction must destroy a small amount of XRP (the transaction cost) to be relayed and included, which deters spam. The cost is the base fee scaled by the current load factor, and rises further under fee escalation when the open ledger is congested. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [Transaction cost on xrpl.org](https://xrpl.org/docs/concepts/transactions/transaction-cost) + + + +### Transaction queue (TxQ) + +The transaction queue (TxQ) holds transactions that pay enough for local relay but not the current open-ledger cost, so the node can include them in a future ledger instead of discarding them. Depth pinned at capacity for sustained periods signals demand exceeding throughput or a fee-spam burst. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Transaction queue (TxQ) on xrpl.org](https://xrpl.org/docs/concepts/transactions/transaction-queue) + + + +## Node State & Sync + + + +### Back-fill / catch-up + +When a node is missing ledgers (at startup, after an outage, or to extend history) it back-fills by fetching them from peers. Elevated back-fill activity is normal while catching up and should fall to near zero once history is complete and the node is synced. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Complete ledger ranges + +A node stores ledger history as one or more contiguous ranges. One continuous range means an unbroken history; many fragmented ranges indicate gaps from missed or failed fetches that the node is still back-filling. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Historical fetch rate + +The rate at which the node fetches older ledgers to extend or repair its stored history. Elevated while back-filling; near zero once history is complete. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Ledger acquire (inbound fetch) + +Acquiring a ledger means requesting it and its contents from peers when the node lacks it. Acquire outcomes split into complete and failed; a rising failed rate means the node cannot fetch needed ledgers from its peers. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Operating mode / server state + +The server state describes how fully the node is participating, in ascending order: disconnected, connected, syncing, tracking, full (caught up), and for validators validating and proposing. A healthy non-validator sits in Full; frequent transitions out of Full indicate instability. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Operating mode / server state on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) + + + +## Peer & Overlay Networking + + + +### Cluster + +A cluster is a set of servers run by the same operator that trust each other, exchanging load and status directly and skipping some redundant checks. Cluster overhead is routine; sustained high cluster overhead suggests frequent cluster-state churn. + +**Scope:** cluster-wide — shared across a co-operated cluster of nodes run by one operator. + +**See also:** [Cluster on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/clustering) + + + +### Fetch-pack + +A fetch-pack is a bulk transfer of ledger data used to accelerate catch-up across a range of ledgers. Fetch-pack traffic rises sharply while catching up and is near zero when fully synced; continuous fetch-pack traffic means the node never fully catches up. + +**Scope:** per node — measured on and specific to this individual server. + + + +### GetObject / object fetch + +GetObject messages fetch individual pieces of ledger data from peers, broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes. Many messages carrying few bytes means small piecemeal fetches; few large messages means batch transfers. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Have / requested transactions + +Have-transaction messages advertise that a peer holds particular transactions; requested-transaction messages ask for them. Comparing requested versus have gauges how well transactions are propagating; requested far exceeding have means peers are behind on propagation. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Insane / diverged peers + +Diverged (insane) peers are connected peers whose reported ledger state does not match the network's. Zero is healthy; a persistent nonzero count can indicate peers on a fork or misbehaving peers. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Ledger tree nodes + +A ledger's contents are stored as Merkle trees (SHAMaps): a transaction tree and an account-state tree, each built from tree nodes. Peers fetch individual tree nodes (tx-node, account-state-node) to reconstruct a ledger. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + + + +### Manifest + +A manifest is a validator's signed statement linking its long-term master key to its current ephemeral signing key, letting it rotate keys without losing trust. Manifest overhead rises around key rotations; sustained high manifest traffic suggests frequent reissue. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + + + +### Overlay + +The overlay is xrpld's peer-to-peer messaging layer connecting nodes. All inter-node traffic (transactions, proposals, validations, ledger data, control messages) flows over it, grouped into traffic categories for accounting. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Overlay on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) + + + +### Proof path + +Proof-path request/response messages let a peer verify an individual ledger entry via its Merkle proof, without downloading the entire ledger. Volume rises when peers verify specific state, often during catch-up. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Reduce-relay + +Reduce-relay lowers overlay bandwidth by forwarding proposals/validations through a chosen subset of peers (selected), suppressing the rest, while some older peers have the feature not-enabled. When suppression hides a message a peer needed, it fetches it on demand; a rising missing-tx rate means suppression is too aggressive. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Replay delta + +Replay-delta request/response messages transfer only the state changes between ledgers so a peer can efficiently replay them during catch-up, rather than refetching whole ledgers. Continuous replay traffic means the node is repeatedly replaying rather than staying current. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Resource disconnect + +The resource manager tracks each peer's load and disconnects those exceeding limits. A rising resource-disconnect count is consistent with abusive or misbehaving peers being shed as backpressure; a flat line is healthy. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Set get/share + +Set-get (fetch) and set-share messages exchange transaction-set data between peers as they reconcile which transactions belong in the closing ledger. Some exchange each ledger is normal; high set-get means peers are frequently missing transaction sets. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Squelch + +Squelching is a relay-control mechanism: a node tells peers to stop sending it a particular validator's messages when it already has a good source, reducing redundant forwarding. High suppressed counts mean squelch is saving bandwidth; ignored directives (peers not honoring squelch) should stay low. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Trusted / untrusted / duplicate + +Proposals and validations are trusted if they come from validators on this node's UNL, untrusted otherwise; duplicates are messages the node already received and discarded. High untrusted volume can indicate trusted-list misconfiguration or spam; high duplicates indicate inefficient relay. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Trusted / untrusted / duplicate on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/unl) + + + +### Validator list + +Validator lists are the signed, published sets of recommended validators (the basis for a node's UNL). Peers exchange them so nodes stay configured with a current trusted set. Traffic bursts when lists update or new peers connect and is otherwise quiet. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [Validator list on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/unl) + + + +## Storage Internals + + + +### Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger) + +xrpld keeps several caches: SLE (ledger entries), Ledger, AcceptedLedger, TreeNode, and the FullBelowCache (subtrees known to be fully present locally). A high hit rate means lookups are served from memory; low or falling hit rates indicate cache thrashing and extra back-end reads. + +**Scope:** per node — measured on and specific to this individual server. + + + +### NodeStore + +The NodeStore is xrpld's content-addressed object database holding all ledger tree nodes, keyed by hash. It is the main on-disk store read during queries and sync and written as new ledgers are stored; NuDB is the default backend. + +**Scope:** per node — measured on and specific to this individual server. + + + +### NuDB + +NuDB is a fast append-only key-value store used as the NodeStore backend. Its on-disk size grows steadily with retained ledger history; the growth slope is the data growth rate. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Read threads / read queue / write load + +The NodeStore serves reads through a pool of read threads (optionally bundling reads) fronted by a read queue, while writes are scored as write load. Read threads pinned at the maximum, or a high read queue, mean read I/O is saturated; high write load means back-end write pressure. + +**Scope:** per node — measured on and specific to this individual server. + + + +## Validator Health + + + +### Amendment blocked + +Amendments are protocol changes activated by validator voting. If the network enables an amendment a node's build does not understand, the node becomes amendment-blocked: it stops processing to avoid diverging, and requires a software upgrade. OK is healthy; BLOCKED means an upgrade is needed. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Amendment blocked on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/amendments) + + + +### Ledgers closed rate + +The rate at which this node closes ledgers, roughly 12-20 per minute on Mainnet (one per ~3-5s close). It should match the network's steady cadence; a drop toward zero means the node stopped participating. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Ledgers closed rate on xrpl.org](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) + + + +### UNL (Unique Node List) + +A node's Unique Node List is the set of validators it trusts not to collude. The node reaches consensus by listening to its UNL and declaring a ledger validated when a quorum of them agree. UNL expiry (days left) matters because an expired list leaves the node without a trusted set. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [UNL (Unique Node List) on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/unl) + + + +### UNL blocked + +UNL-blocked means the node could not establish a valid trusted validator list (for example, all configured lists expired or failed to load), so validator trust cannot be established. OK is healthy; BLOCKED halts safe participation. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [UNL blocked on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol) + + + +### Validation agreement + +Validation agreement is the fraction of recent ledgers where this validator issued a validation matching the consensus outcome (agreed) rather than missing or disagreeing (missed). A sustained dip signals configuration drift, unreliability, or a network partition. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Validation agreement on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) + + + +### Validation quorum + +The quorum is the minimum count of trusted-validator validations that must agree before a server declares a ledger validated (by default about 80% of the UNL). It is derived from the active validator list and changes when that list changes. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [Validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) + + + +### Validations checked vs sent + +Checked validations are those received from other validators and verified by this node (reflecting network validation traffic reaching it); sent validations are those this node issues (roughly one per closed ledger for a validator). + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Validations checked vs sent on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) + + + +## RPC & Pathfinding + + + +### Batch vs single RPC + +A single request carries one command; a batch request bundles several. Single requests usually dominate; a batch rate climbing sharply is consistent with bulk automation or amplification attempts. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Clio / reporting client + +Clio is a separate reporting server that offloads historical and API queries from xrpld, fetching data via the gRPC interface. gRPC panels are nonzero mainly when reporting/Clio-style clients are connected. + +**Scope:** per node — measured on and specific to this individual server. + + + +### gRPC + +gRPC is a high-performance binary RPC protocol the node exposes for specific consumers, chiefly reporting-mode (Clio) clients. gRPC traffic is zero on nodes without such clients; its status is only ever success or error. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Order book + +An order book holds the standing offers to exchange a given currency pair in the XRP Ledger's decentralized exchange. Pathfinding and some RPC queries walk order books; cost grows with order-book depth and request complexity. + +**Scope:** network-wide — a protocol-shared value, the same across all nodes. + +**See also:** [Order book on xrpl.org](https://xrpl.org/docs/concepts/tokens/decentralized-exchange) + + + +### Path request / discovery + +A path request is a client subscription for payment paths; discovery passes are the periodic refreshes the node runs to keep those paths current as ledgers close. Discovery cost tracks request demand and is a cost driver for subscription-heavy nodes. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Path request / discovery on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods/path-and-order-book-methods/path_find) + + + +### Pathfinding (fast / full) + +Pathfinding searches for routes along which a cross-currency payment can flow through order books and AMMs. A fast search trades accuracy for speed; a full search is exhaustive and much more expensive. Sustained high durations indicate pathfinding-heavy clients straining the node. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Pathfinding (fast / full) on xrpl.org](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) + + + +### Resource drops / warnings + +The resource manager meters each peer and client; it first warns endpoints for excessive usage and then drops or blocks them. Nonzero rates mean the node is actively rejecting abusive connections; zero is expected when no abusive consumers are present. + +**Scope:** per node — measured on and specific to this individual server. + + + +### RPC command / method + +Clients call the node via named RPC commands (also called methods), such as account_info, ledger, or submit, over HTTP, WebSocket, or gRPC. Panels break rate, latency, and errors down by command; heavy commands (ledger/account queries) cost far more than status calls. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [RPC command / method on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) + + + +### WebSocket + +WebSocket is a long-lived connection transport for the node's API, used by clients that subscribe to streams or send many requests. WebSocket message rate is nonzero only when clients use WebSocket; HTTP-only nodes read zero. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [WebSocket on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/api-conventions) diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md new file mode 100644 index 00000000000..6ec99c27a8d --- /dev/null +++ b/docs/telemetry-runbook.md @@ -0,0 +1,2427 @@ +# xrpld Telemetry Operator Runbook + +## Table of Contents + +- [Overview](#overview) +- [Quick Start](#quick-start) +- [Configuration Reference](#configuration-reference) +- [Exporting to Grafana Cloud](#exporting-to-grafana-cloud) +- [Span Reference](#span-reference) + - [RPC Spans](#rpc-spans) + - [Transaction Spans](#transaction-spans) + - [Transaction Queue Spans](#transaction-queue-spans) + - [PathFinding Spans](#pathfinding-spans) + - [Consensus Spans](#consensus-spans) + - [Ledger Spans](#ledger-spans) + - [Peer Spans](#peer-spans) +- [Protocol Span Flow](#protocol-span-flow) + - [Master overview](#master-overview) + - [Client and peer ingress](#client-and-peer-ingress) + - [Shared transaction apply pipeline](#shared-transaction-apply-pipeline) + - [Consensus round](#consensus-round) + - [Accept, build, and finalize the ledger](#accept-build-and-finalize-the-ledger) + - [Side flows: pathfinding and ledger acquire](#side-flows-pathfinding-and-ledger-acquire) + - [Where telemetry parenting differs from protocol flow](#where-telemetry-parenting-differs-from-protocol-flow) +- [Insights and Sample Queries](#insights-and-sample-queries) + - [Transaction Workflow Analysis](#transaction-workflow-analysis) + - [DEX (OfferCreate / OfferCancel)](#dex-offercreate--offercancel) + - [Apply Pipeline by Stage](#apply-pipeline-by-stage) + - [Transaction Queue Health](#transaction-queue-health) + - [RPC Debugging](#rpc-debugging) + - [PathFinding Performance](#pathfinding-performance) + - [Consensus Health](#consensus-health) + - [Cross-Subsystem Correlation](#cross-subsystem-correlation) +- [Cross-Node Trace Propagation](#cross-node-trace-propagation) +- [Prometheus Metrics (Spanmetrics)](#prometheus-metrics-spanmetrics) +- [System Metrics (OTel native -- beast::insight)](#system-metrics-otel-native----beastinsight) +- [Grafana Dashboards](#grafana-dashboards) +- [Alerting](#alerting) +- [Log-Trace Correlation](#log-trace-correlation) +- [Troubleshooting](#troubleshooting) +- [Performance Tuning](#performance-tuning) +- [Disabling Telemetry](#disabling-telemetry) +- [Validating Telemetry Stack](#validating-telemetry-stack) +- [Performance Benchmarking](#performance-benchmarking) + +## Overview + +xrpld supports OpenTelemetry distributed tracing to provide visibility into RPC requests, transaction processing, and consensus rounds. + +This runbook covers operating a running node and querying its traces. For +building xrpld with telemetry support and the internal architecture, see +[build/telemetry.md](build/telemetry.md). For plain-language definitions of the +XRP Ledger terms used in the dashboards, see the +[telemetry glossary](telemetry-glossary.md). + +## Quick Start + +### 1. Start the observability stack + +```bash +docker compose -f docker/telemetry/docker-compose.yml up -d +``` + +This starts: + +- **OTel Collector** on ports 4317 (gRPC) and 4318 (HTTP), and 13133 (health) +- **Tempo** on http://localhost:3200 (trace backend) +- **Prometheus** on http://localhost:9090 +- **Loki** on http://localhost:3100 (log aggregation) +- **Grafana** on http://localhost:3000 (Tempo pre-configured as datasource) + +### 2. Enable telemetry in xrpld + +Add to your `xrpld.cfg`: + +```ini +[telemetry] +enabled=1 +endpoint=http://localhost:4318/v1/traces +``` + +### 3. Build with telemetry support + +```bash +conan install . --build=missing -o telemetry=True +cmake --preset default -Dtelemetry=ON +cmake --build --preset default +``` + +### 4. Run against a live network + +Two ready-made configs connect a tracking node (no validator credentials) to a +public network with all tracing and native metrics enabled: + +| Config | Network | +| ---------------------------------------------- | ------- | +| `docker/telemetry/xrpld-telemetry.cfg` | Devnet | +| `docker/telemetry/xrpld-telemetry-mainnet.cfg` | Mainnet | + +```bash +.build/xrpld --conf docker/telemetry/xrpld-telemetry-mainnet.cfg +``` + +Both set `[insight] server=otel` (native metrics → collector → Prometheus, which +drives the dashboards) and `service_instance_id`, exposed by Prometheus as the +`service_instance_id` label that the `$node` dashboard variable filters on. The +mainnet config logs to `/var/log/xrpld/mainnet/debug.log` — the path +the collector's filelog receiver tails for log-trace correlation. + +Metrics begin flowing as soon as the node connects to peers (`server_state` +≥ `connected`); full ledger and consensus panels populate after sync +(`server_state` = `full`). Check progress with: + +```bash +curl -s http://localhost:5005 -d '{"method":"server_info"}' | + jq '.result.info | {server_state, peers, complete_ledgers}' +``` + +> Mainnet sync is bandwidth- and disk-heavy. For a quick check use the devnet +> config or the standalone test in `docker/telemetry/TESTING.md`, which +> generates spans without waiting for a live sync. + +## Configuration Reference + +| Option | Default | Description | +| -------------------------- | --------------------------------- | --------------------------------------------------------- | +| `enabled` | `0` | Master switch for telemetry | +| `endpoint` | `http://localhost:4318/v1/traces` | OTLP/HTTP endpoint | +| `service_name` | `xrpld` | OpenTelemetry service name resource attribute | +| `service_instance_id` | node public key | OpenTelemetry service instance ID resource attribute | +| `trace_rpc` | `1` | Enable RPC request tracing | +| `trace_transactions` | `1` | Enable transaction tracing | +| `trace_consensus` | `1` | Enable consensus tracing | +| `trace_peer` | `1` | Enable peer message tracing (high volume) | +| `trace_ledger` | `1` | Enable ledger tracing | +| `consensus_trace_strategy` | `deterministic` | Consensus trace ID strategy (`deterministic` or `random`) | +| `batch_size` | `512` | Max spans per batch export | +| `batch_delay_ms` | `5000` | Delay between batch exports | +| `max_queue_size` | `2048` | Max spans queued before dropping | +| `use_tls` | `0` | Use TLS for exporter connection | +| `tls_ca_cert` | (empty) | Path to CA certificate bundle | +| `tls_client_cert` | (empty) | Client cert (PEM) for mutual TLS; empty = one-way TLS | +| `tls_client_key` | (empty) | Private key (PEM) for `tls_client_cert` | + +## Exporting to Grafana Cloud + +The collector can ship traces, metrics, and logs to a hosted **Grafana +Cloud** stack instead of (or alongside) the local Tempo/Prometheus/Loki +backends. This is a runtime choice — no xrpld rebuild and no change to the +base stack. xrpld still exports to the local collector exactly as before; +the collector adds one OTLP/HTTP exporter that forwards all three signals to +the Grafana Cloud OTLP gateway, which fans them out to hosted Tempo, Mimir, +and Loki. + +### Credentials + +Find these under **Grafana Cloud → Connections → OpenTelemetry (OTLP)**: + +| Value | Used as | Notes | +| ----------------------------- | ----------------- | -------------------------------------------------- | +| `GRAFANA_CLOUD_OTLP_ENDPOINT` | exporter endpoint | Full gateway URL incl. `/otlp` path | +| `GRAFANA_CLOUD_INSTANCE_ID` | Basic-auth user | Numeric stack/instance id | +| `GRAFANA_CLOUD_API_TOKEN` | Basic-auth pass | Access-policy token with `*:write` for all signals | + +### Enable + +1. Copy the template and fill in the three values: + + ```bash + cp docker/telemetry/.env.grafanacloud.example docker/telemetry/.env.grafanacloud + # edit .env.grafanacloud — this file is gitignored, never commit tokens + ``` + +2. Bring the stack up with the base file **and** the Grafana Cloud override: + + ```bash + docker compose -f docker/telemetry/docker-compose.yml \ + -f docker/telemetry/docker-compose.grafanacloud.yaml up -d + ``` + +To return to local-only export, bring the stack up with just the base +`docker-compose.yml`. + +### Files + +| File | Role | +| ----------------------------------------- | ---------------------------------------------------------------------------------------------- | +| `otel-collector-config.grafanacloud.yaml` | Collector config: local backends **plus** a Grafana Cloud OTLP exporter on all three pipelines | +| `docker-compose.grafanacloud.yaml` | Override that mounts that config and injects the credentials | +| `.env.grafanacloud.example` | Credential template (copy to `.env.grafanacloud`) | + +### Local + cloud vs cloud-only + +The prepared config **dual-exports**: data goes to both the local stack and +Grafana Cloud, so the on-box backends remain a fallback. For cloud-only, +remove the local exporters (`debug`, `otlp/tempo`, `prometheus`, +`otlphttp/loki`) from the respective pipelines in +`otel-collector-config.grafanacloud.yaml`, leaving only +`otlphttp/grafanacloud`. + +> **Note**: shipping logs to Grafana Cloud requires keeping xrpld file +> logging on (at least `warning` level) so the collector's filelog receiver +> has a `debug.log` to tail. Traces and metrics are unaffected by log level. + +### Importing dashboards to Grafana Cloud + +Shipping data (above) is independent of installing the dashboards. The local +stack auto-provisions dashboards from a mounted folder +(`grafana/provisioning/dashboards/dashboards.yaml`, `type: file`); Grafana +Cloud cannot read your filesystem, so its dashboards must be imported over the +HTTP API or the UI. + +The dashboard JSON in `docker/telemetry/grafana/dashboards/` references its +backends through datasource **template variables** (`${DS_PROMETHEUS}`, +`${DS_TEMPO}`) rather than fixed UIDs. On import, Grafana binds each variable +to a datasource of the matching type — auto-selecting it when only one exists +(the usual case: one Mimir, one Tempo). This is what makes the same files work +unchanged on both the local stack and Cloud. + +> Dashboards are parameterized by `grafana/parameterize-datasources.py`. If you +> add a dashboard exported with hardcoded UIDs, re-run that script (idempotent) +> before committing so it stays portable. + +To import: + +1. In Grafana Cloud, go to **Dashboards → New → Import**. +2. Upload a file from `docker/telemetry/grafana/dashboards/` (or paste its + JSON), then click **Load**. +3. At the datasource prompt, confirm the auto-selected **Prometheus/Mimir** + datasource — and **Tempo** for dashboards that query traces — then + **Import**. +4. Repeat per dashboard. Only `consensus-health` uses Tempo; the rest need + only the Prometheus/Mimir datasource. + +## Span Reference + +All spans instrumented in xrpld, grouped by subsystem: + +### RPC Spans + +| Span Name | Source File | Attributes | Description | +| -------------------- | ----------------- | ----------------------------------------------------------- | ----------------------------------------------------- | +| `rpc.http_request` | ServerHandler.cpp | `request_payload_size` | Top-level HTTP RPC request | +| `rpc.ws_upgrade` | ServerHandler.cpp | — | WebSocket upgrade handshake | +| `rpc.ws_message` | ServerHandler.cpp | `command` | WebSocket RPC message | +| `rpc.process` | ServerHandler.cpp | `is_batch`, `batch_size` | RPC processing (child of rpc.http_request/ws_message) | +| `rpc.command.` | RPCHandler.cpp | `command`, `version`, `rpc_role`, `rpc_status`, `load_type` | Per-command span (e.g., `rpc.command.server_info`) | + +### Transaction Spans + +| Span Name | Source File | Attributes | Description | +| --------------- | --------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------- | +| `tx.process` | NetworkOPs.cpp | `tx_hash`, `local`, `path`, `tx_type`, `fee`, `sequence`, `ter_result`, `applied`, `current_ledger_seq` | Transaction submission and processing | +| `tx.receive` | PeerImp.cpp | `peer_id`, `tx_hash`, `tx_type`, `peer_version`, `suppressed`, `tx_status`, `current_ledger_seq` | Transaction received from peer relay | +| `tx.apply` | BuildLedger.cpp | `ledger_seq`, `tx_count`, `tx_failed` | Transaction set applied per ledger | +| `tx.preflight` | applySteps.cpp | `stage`, `tx_type`, `ter_result` | Stateless checks stage | +| `tx.preclaim` | applySteps.cpp | `stage`, `tx_type`, `ter_result`, `current_ledger_seq`, `current_ledger_hash` | Ledger-aware checks stage | +| `tx.transactor` | Transactor.cpp | `stage`, `tx_type`, `ter_result`, `applied`, `current_ledger_seq`, `current_ledger_hash` | Apply stage (transactor runs) | + +The three apply-pipeline spans (`tx.preflight`, `tx.preclaim`, `tx.transactor`) +share a deterministic `trace_id` from `txID[0:16]`, so they group under one +trace per transaction. The `stage` attribute (`preflight` / `preclaim` / +`apply`) drives the collector spanmetrics `stage` dimension, giving per-stage +RED metrics on the _Transaction Overview_ dashboard. + +`current_ledger_seq` is the current (open/in-flight) ledger index a span acted on +— the ledger being worked on, not an established one — so a transaction's +txID-keyed spans can be joined to the ledger trace it targeted +(`span.current_ledger_seq`). The view-bearing stages (`tx.preclaim`, +`tx.transactor`) also carry `current_ledger_hash` (the current ledger's parent +hash); `tx.preflight` is stateless and omits both. + +### Transaction Queue Spans + +| Span Name | Source File | Attributes | Description | +| ------------------ | ----------- | ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `txq.enqueue` | TxQ.cpp | `tx_hash`, `tx_type`, `current_ledger_seq`, `current_ledger_hash` | Enqueue decision; parents to `tx.process` on the submission path (explicit context), a root on the open-ledger rebuild path — `current_ledger_seq` correlates it to the ledger in both cases | +| `txq.apply_direct` | TxQ.cpp | -- | Direct apply attempt (bypassing queue) | +| `txq.batch_clear` | TxQ.cpp | -- | Batch clear of queued transactions for an account | +| `txq.accept` | TxQ.cpp | `queue_size`, `ledger_changed` | Ledger-close accept loop over queued transactions | +| `txq.accept_tx` | TxQ.cpp | `tx_hash`, `retries_remaining`, `ter_code`, `txq_status` | Per-transaction apply during accept | +| `txq.cleanup` | TxQ.cpp | `ledger_seq` | Post-close cleanup of expired queue entries | + +### PathFinding Spans + +| Span Name | Source File | Attributes | Description | +| --------------------- | --------------------------------- | -------------------------------------------------- | ------------------------------------------------------- | +| `pathfind.request` | PathFind.cpp / RipplePathFind.cpp | `pathfind_source_account`, `pathfind_dest_account` | Path-find RPC entry (accounts hashed; set when present) | +| `pathfind.compute` | PathRequest.cpp | `pathfind_fast`, `pathfind_dest_currency` | Path computation for one request (`doUpdate`) | +| `pathfind.discover` | PathRequest.cpp | `pathfind_search_level`, `pathfind_num_paths` | Graph exploration (one per RPC call in `findPaths`) | +| `pathfind.update_all` | PathRequestManager.cpp | `pathfind_ledger_index`, `pathfind_num_requests` | Async recomputation of active requests on ledger close | + +### Consensus Spans + +| Span Name | Source File | Attributes | Description | +| ------------------------------ | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `consensus.round` | RCLConsensus.cpp | `consensus_ledger_id`, `ledger_seq`, `consensus_mode`, `trace_strategy`, `consensus_round_id` | Root span for a consensus round (deterministic or random trace ID) | +| `consensus.phase.open` | Consensus.h | -- | Open phase duration (child of round) | +| `consensus.proposal.send` | RCLConsensus.cpp | `consensus_round`, `is_bow_out` | Consensus proposal broadcast | +| `consensus.ledger_close` | RCLConsensus.cpp | `ledger_seq`, `consensus_mode` | Ledger close event | +| `consensus.establish` | Consensus.h | `converge_percent`, `establish_count`, `proposers` | Establish phase duration (child of round) | +| `consensus.update_positions` | Consensus.h | `converge_percent`, `proposers`, `disputes_count` | Position update and dispute resolution (see Events below) | +| `consensus.check` | Consensus.h | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `proposers_finished`, `consensus_stalled`, `establish_count`, `consensus_result` | Consensus threshold check | +| `consensus.accept` | RCLConsensus.cpp | `proposers`, `round_time_ms`, `quorum`, `disputes_count`, `consensus_state` | Ledger accepted by consensus | +| `consensus.accept.apply` | RCLConsensus.cpp | `ledger_seq`, `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction`, `tx_count` | Ledger application with close time details (see Events below) | +| `consensus.validation.send` | RCLConsensus.cpp | `ledger_seq`, `proposing`, `ledger_hash`, `full_validation`, `validation_sign_time` | Validation sent after accept (follows-from link) | +| `consensus.mode_change` | RCLConsensus.cpp | `mode_old`, `mode_new` | Consensus mode transition | +| `consensus.proposal.receive` | PeerImp.cpp | `proposal_trusted`, `consensus_round` | Proposal received from peer (extracts parent context from TraceContext when present; falls back to standalone span for older peers) | +| `consensus.validation.receive` | PeerImp.cpp | `validation_trusted`, `ledger_seq` | Validation received from peer (extracts parent context from TraceContext when present; falls back to standalone span for older peers) | + +#### Consensus Span Events + +| Parent Span | Event Name | Event Attributes | Description | +| ---------------------------- | ----------------- | ----------------------------------------------------------- | ------------------------------------------------------- | +| `consensus.update_positions` | `dispute.resolve` | `tx_id`, `dispute_our_vote`, `dispute_yays`, `dispute_nays` | Emitted per dispute when votes are tallied | +| `consensus.accept.apply` | `tx.included` | `tx_id` | Emitted per transaction included in the accepted ledger | + +#### Close Time Queries (Tempo TraceQL) + +Span attributes are filtered with `span.` inside `{}`. Combine conditions with `&&`. + +``` +# Find rounds where validators disagreed on close time +{name="consensus.accept.apply" && span.close_time_correct = false} + +# Find consensus failures (moved_on) +{name="consensus.accept.apply" && span.consensus_state = "moved_on"} + +# Find slow ledger applications (>5s) +{name="consensus.accept.apply" && duration > 5000ms} + +# Find specific ledger's consensus details +{name="consensus.accept.apply" && span.ledger_seq = 92345678} + +# Find all spans in a consensus round (deterministic trace strategy) +{name="consensus.round" && span.consensus_round_id = ""} + +# Find dispute resolutions +{name="consensus.update_positions"} >> {event:name="dispute.resolve"} +``` + +### Ledger Spans + +| Span Name | Source File | Attributes | Description | +| ----------------- | -------------------- | ------------------------------------- | ----------------------------- | +| `ledger.build` | BuildLedger.cpp:31 | `ledger_seq`, `tx_count`, `tx_failed` | Ledger build during consensus | +| `ledger.validate` | LedgerMaster.cpp:915 | `ledger_seq`, `validations` | Ledger promoted to validated | +| `ledger.store` | LedgerMaster.cpp:409 | `ledger_seq` | Ledger stored in history | + +### Peer Spans + +| Span Name | Source File | Attributes | Description | +| ------------------------- | ---------------- | ------------------------------- | ----------------------------- | +| `peer.proposal.receive` | PeerImp.cpp:1667 | `peer_id`, `proposal_trusted` | Proposal received from peer | +| `peer.validation.receive` | PeerImp.cpp:2264 | `peer_id`, `validation_trusted` | Validation received from peer | + +Both peer receive spans are `kConsumer` inbound entry points started as fresh +trace roots. They never inherit an ambient span left active on the peer thread, +so they do not nest under an unrelated transaction's trace. The distributed +child span that links back to the sending node is the separate +`consensus.*.receive` / `tx.receive` span (see Cross-Node Trace Propagation). + +--- + +## Protocol Span Flow + +This section maps every span type onto the **real xrpld control flow and XRPL +protocol order** (verified against code and [docs/consensus.md](consensus.md)) — +what the code actually executes next, in what order, with which loops and +branches. Spans are drawn as **labels on real operations**, not as their +OpenTelemetry parent links; the SDK's span parenting is listed separately in +[Where telemetry parenting differs from protocol flow](#where-telemetry-parenting-differs-from-protocol-flow). + +These diagrams are the **canonical key for linking the span hierarchy** — every +node and every branch is labelled with the span that represents that state or +transition, so a span can be wired to its true protocol parent/child by reading +the graph. They are therefore drawn **exact, not simplified**: every real loop, +retry, recovery, and drop branch is shown even when it adds clutter. + +Naming and edge conventions: + +- **Rectangle `[ ]`** — a state/operation that **emits a span**; the first line is + the exact `span.name`, the parenthetical below is the operation. +- **Rounded `( )` with `(no span)`** — a real protocol step that emits **no span**; + shown so the flow stays continuous and is never mistaken for a missing span. +- **Solid arrow** — the code calls or sequences directly into the next operation. + When the transition itself emits a span, the edge is labelled `→[span.name]`; + otherwise it carries the branch condition. +- **`↻`** — the edge repeats (per tx, per peer, per dispute, per pass, per round). +- **Dashed arrow** — a conditional branch or an async job hand-off. +- **Dotted `⇢ ctx`** — trace context crosses a node boundary over a protobuf peer + message (`sender.span ⇢ receiver.span`); a different node continues the trace — + **not** an in-process call. +- **Red-bordered node** — a terminal **drop / abandon** state. + +### Master overview + +Five ingress origins feed two shared engines — the per-transaction **apply +pipeline** and the **consensus round** — which converge on ledger build → store → +validate. Pathfinding and ledger-acquire are side flows. A single ledger can take +**many consensus rounds** to settle (see [Consensus round](#consensus-round)). + +```mermaid +flowchart TB + classDef ingress fill:#1d4ed8,stroke:#1e3a8a,color:#fff; + classDef engine fill:#047857,stroke:#064e3b,color:#fff; + classDef consensus fill:#b45309,stroke:#7c2d12,color:#fff; + classDef ledger fill:#6d28d9,stroke:#4c1d95,color:#fff; + classDef side fill:#0e7490,stroke:#155e75,color:#fff; + classDef plain fill:#334155,stroke:#0f172a,color:#fff; + + subgraph ING["Ingress (protocol entry points)"] + direction TB + RPC["rpc.http_request / rpc.ws_message
rpc.ws_upgrade / grpc.MethodName
(client transport in)"]:::ingress + SUB(["submit command
(no span)"]):::plain + PRELAY["tx.receive
(peer relay in)"]:::ingress + PMSG["peer.proposal.receive
peer.validation.receive
(peer overlay in)"]:::ingress + end + + TXP["tx.process
(NetworkOPs::processTransaction)"]:::engine + OPEN["txq.enqueue
(open-ledger apply + TxQ decision)"]:::engine + PIPE["tx.preflight → tx.preclaim → tx.transactor
(SHARED apply pipeline)"]:::engine + + subgraph CONS["Consensus round"] + direction TB + ROUND["consensus.round
(Open → Establish → Accepted)"]:::consensus + ACC["consensus.accept → consensus.accept.apply"]:::consensus + end + + subgraph LGR["Ledger finalize"] + direction TB + BUILD["ledger.build
(tx.apply over agreed set)"]:::ledger + STORE["ledger.store
(built, NOT yet final)"]:::ledger + VAL["ledger.validate
(promoted at quorum)"]:::ledger + end + + subgraph SIDE["Side flows"] + direction TB + PF["pathfind.update_all
pathfind.request/compute/discover"]:::side + ACQ["ledger.acquire
(fetch missing / correct prior)"]:::side + end + + RPC -.->|submit / submit_multisigned| SUB --> TXP + RPC -.->|path_find / ripple_path_find| PF + PRELAY --> TXP + TXP --> OPEN + OPEN -->|↻ up to 3 passes| PIPE + OPEN -.->|txq.accept re-apply queued tx each close ↻| PIPE + PMSG -->|peerProposal / recvValidation| ROUND + ROUND --> ACC --> BUILD + BUILD -->|↻ each tx × up to 3 passes| PIPE + BUILD --> STORE + PMSG -. "trusted validations arrive async → checkAccept quorum" .-> VAL + ROUND -. "avalanche rounds ↻ (threshold 50→65→70→95%)" .-> ROUND + ACC -->|endConsensus ↻ next round until a ledger validates| ROUND + ROUND -.->|wrong-ledger: request correct prior| ACQ + ACQ -.->|switch-ledger: resume round on correct prior| ROUND + ACQ --> STORE + VAL -.->|missing ledger| ACQ + BUILD -.->|every close re-runs| PF +``` + +### Client and peer ingress + +RPC submit and peer relay **converge** at `tx.process`, the single NetworkOPs +entry. gRPC serves ledger queries only — it has no submit path and never runs +`doCommand`. + +```mermaid +flowchart TB + classDef span fill:#1d4ed8,stroke:#1e3a8a,color:#fff; + classDef plain fill:#334155,stroke:#0f172a,color:#fff; + classDef drop fill:#7f1d1d,stroke:#ef4444,color:#fff; + + HTTP["rpc.http_request
(HTTP entry)"]:::span + PROC["rpc.process
(parse + batch)"]:::span + CMD["rpc.command.NAME
(one command)"]:::span + WSU["rpc.ws_upgrade
(WS handshake)"]:::span + WSM["rpc.ws_message
(one frame)"]:::span + GRPC["grpc.MethodName
(ledger query)"]:::span + GH(["handler_ ctx
(no span)"]):::plain + SUBMIT(["doSubmit
(no span)"]):::plain + TXP["tx.process
(NetworkOPs::processTransaction)"]:::span + RELAYOUT(["Overlay::relay fan-out to N peers
(no span; if applied / terQUEUED,
shouldRelay, not tfInnerBatchTxn)"]):::plain + PREDROP(["Diverged / needNetworkLedger
(no span — dropped before tx.receive)"]):::drop + RCV["tx.receive
(peer TMTransaction in)"]:::span + RCVDROP["tx.receive
tx_status = rejected_inner_batch /
suppressed / dropped_no_sync /
dropped_queue_full"]:::drop + CHK(["checkTransaction
(JtTransaction worker, no span)"]):::plain + PRELAY_IN(["TMTransaction in (no span)"]):::plain + + HTTP -->|processRequest| PROC + PROC -->|↻ each batch request → doCommand| CMD + WSU -. "each inbound frame → onWSMessage" .-> WSM + WSM -->|doCommand| CMD + GRPC --> GH + CMD -.->|submit / submit_multisigned| SUBMIT + SUBMIT -->|processTransaction| TXP + TXP -.->|relay applied / queued tx| RELAYOUT + + PRELAY_IN -.->|tracking == Diverged / needNetworkLedger| PREDROP + PRELAY_IN -->|else| RCV + RCV -.->|inner-batch / dup / age>4min / JtTransaction full| RCVDROP + RCV -->|addJob JtTransaction| CHK + CHK -->|processTransaction, trusted=peer| TXP + + RELAYOUT -. "tx.process ⇢ tx.receive (span_id over TMTransaction)" .-> RCV +``` + +Ingress branches (all evidence in code): + +- `onHandoff`: WS upgrade vs peer bundle vs status page vs legacy HTTP + ([ServerHandler.cpp:227](../src/xrpld/rpc/detail/ServerHandler.cpp#L227)). +- `doSubmit`: `tx_blob` present → submit signed blob; absent → server + sign-and-submit ([Submit.cpp:49](../src/xrpld/rpc/handlers/transaction/Submit.cpp#L49)). +- `tx.process`: local RPC → `doTransactionSync`; peer → `doTransactionAsync` + (JtBatch) ([NetworkOPs.cpp:1434](../src/xrpld/app/misc/NetworkOPs.cpp#L1434)). +- **Pre-span peer drops** (no `tx.receive` created): `Diverged` + ([PeerImp.cpp:1299](../src/xrpld/overlay/detail/PeerImp.cpp#L1299)) / + `needNetworkLedger` ([1302](../src/xrpld/overlay/detail/PeerImp.cpp#L1302)), + before the span at ~1320. +- **Post-span peer drops** (span exists, `tx_status` set, no job enqueued): + `tfInnerBatchTxn` ([1348](../src/xrpld/overlay/detail/PeerImp.cpp#L1348)), + HashRouter dup/`BAD` ([1361](../src/xrpld/overlay/detail/PeerImp.cpp#L1361)), + `dropped_no_sync` when validated-ledger age > 4 min + ([1416](../src/xrpld/overlay/detail/PeerImp.cpp#L1416)), `dropped_queue_full` + when `JtTransaction` jobs > `maxTransactions` + ([1421](../src/xrpld/overlay/detail/PeerImp.cpp#L1421)). +- **Relay fan-out**: an accepted/queued `tx.process` relays to N peers via + `Overlay::relay`, gated on `applied || (non-FULL local) || terQUEUED`, + HashRouter `shouldRelay`, and not `tfInnerBatchTxn`; the span context is + injected here ([NetworkOPs.cpp:1797](../src/xrpld/app/misc/NetworkOPs.cpp#L1797)). + +Inbound consensus messages take a two-stage handler — a fresh-root `peer.*.receive` +span created first (kConsumer, always), then a `consensus.*.receive` span (only if +not dropped) that carries the sender's context — before enqueuing a `checkPropose` +/ `checkValidation` worker job. The drop points are **asymmetric**: proposals drop +entirely before `consensus.proposal.receive`, while validations can drop both +before and after `consensus.validation.receive`. + +```mermaid +flowchart TB + classDef span fill:#0e7490,stroke:#155e75,color:#fff; + classDef plain fill:#334155,stroke:#0f172a,color:#fff; + classDef drop fill:#7f1d1d,stroke:#ef4444,color:#fff; + + PPR["peer.proposal.receive
(freshRoot, always)"]:::span + PPRDROP(["no consensus.proposal.receive
(untrusted+relay-off / dup /
untrusted+Diverged / untrusted+loaded)"]):::drop + CPR["consensus.proposal.receive
(carries sender ctx)"]:::span + CP(["checkPropose worker
(no span)"]):::plain + SIGP(["sig-fail: charge, drop
(no relay, no span)"]):::drop + PTP(["processTrustedProposal → peerProposal
(no span)"]):::plain + + PVR["peer.validation.receive
(freshRoot, always)"]:::span + pvrDrop1(["no consensus.validation.receive
(!isCurrent / relay-off / dup)"]):::drop + CVR["consensus.validation.receive
(carries sender ctx)"]:::span + cvrDrop2(["dropped after span
(untrusted+Diverged /
untrusted+loaded → no job/relay)"]):::drop + CV(["checkValidation worker
(no span)"]):::plain + SIGV(["!isValid: charge, drop
(no span)"]):::drop + RV(["recvValidation → handleNewValidation
(no span)"]):::plain + RELAY(["Overlay::relay fan-out to N peers
(no span)"]):::plain + + PPR -.->|4 drop conditions| PPRDROP + PPR -->|else| CPR + CPR -->|addJob JtProposalT/Ut| CP + CP -.->|!checkSign| SIGP + CP -->|isTrusted| PTP + CP -.->|if relay| RELAY + + PVR -.->|3 drop conditions| pvrDrop1 + PVR -->|else| CVR + CVR -.->|untrusted+Diverged / loaded| cvrDrop2 + CVR -->|addJob JtValidationT/Ut| CV + CV -.->|!isValid| SIGV + CV -->|recvValidation| RV + CV -.->|if relay / cluster| RELAY +``` + +Consensus-message drop evidence: + +- Both `peer.proposal.receive` and `peer.validation.receive` are `freshRoot` + spans created at the top of `onMessage` + ([PeerImp.cpp:1766](../src/xrpld/overlay/detail/PeerImp.cpp#L1766), + [2389](../src/xrpld/overlay/detail/PeerImp.cpp#L2389)) — so they exist even for + dropped messages. +- **Proposal drops (all before `consensus.proposal.receive` at + [1868](../src/xrpld/overlay/detail/PeerImp.cpp#L1868))**: untrusted+relay-off + ([1807](../src/xrpld/overlay/detail/PeerImp.cpp#L1807)), duplicate + ([1832](../src/xrpld/overlay/detail/PeerImp.cpp#L1832)), untrusted+Diverged + ([1840](../src/xrpld/overlay/detail/PeerImp.cpp#L1840)), untrusted+loaded + ([1846](../src/xrpld/overlay/detail/PeerImp.cpp#L1846)). +- **Validation drops (asymmetric around `consensus.validation.receive` at + [2476](../src/xrpld/overlay/detail/PeerImp.cpp#L2476))**: before — `!isCurrent` + ([2426](../src/xrpld/overlay/detail/PeerImp.cpp#L2426)), relay-off + ([2445](../src/xrpld/overlay/detail/PeerImp.cpp#L2445)), duplicate + ([2468](../src/xrpld/overlay/detail/PeerImp.cpp#L2468)); after — untrusted+Diverged + ([2489](../src/xrpld/overlay/detail/PeerImp.cpp#L2489)), untrusted+loaded + ([2506](../src/xrpld/overlay/detail/PeerImp.cpp#L2506)). +- **Worker sig-fail drops** (charged `kFeeInvalidSignature`, suppress processing + and relay): `checkPropose !checkSign` + ([PeerImp.cpp:3105](../src/xrpld/overlay/detail/PeerImp.cpp#L3105)), + `checkValidation !isValid` + ([3149](../src/xrpld/overlay/detail/PeerImp.cpp#L3149)). + +### Shared transaction apply pipeline + +The apply pipeline is the **single protocol tx-processing chain**, expressed in +code as one composed call +([apply.cpp:118](../src/libxrpl/tx/apply.cpp#L118)): +`doApply(preclaim(preflight(), …), …)`. C++ evaluates inner-to-outer, so +`preflight` runs first, feeds `preclaim`, which feeds `doApply`. Each stage +inspects the prior stage's `TER` and no-ops if it already failed. + +**Four invokers** point into this one pipeline; the diagram draws it once. + +```mermaid +flowchart TB + classDef span fill:#047857,stroke:#064e3b,color:#fff; + classDef plain fill:#334155,stroke:#0f172a,color:#fff; + classDef inv fill:#1d4ed8,stroke:#1e3a8a,color:#fff; + classDef drop fill:#7f1d1d,stroke:#ef4444,color:#fff; + + I1(["open-ledger applyOne (no span)
↻ up to 3 passes"]):::inv + I2["txq.apply_direct
(fee ≥ required)"]:::span + I3["txq.batch_clear / txq.accept_tx
↻ per queued tx"]:::span + I4["tx.apply
(consensus set, ↻ each tx × 3 passes)"]:::span + + REPF(["txq path: rules/flags changed?
re-run preflight (no span)"]):::plain + FREE(["xrpl::apply() (no span)"]):::plain + PF["tx.preflight
(stateless checks)"]:::span + PC["tx.preclaim
(ledger-aware checks)"]:::span + TR["tx.transactor
(mutate stage)"]:::span + CLS(["classify final TER (no span)"]):::plain + OK(["Success / erase (no span)"]):::plain + FAIL(["tef / tem / tel → hard fail, erase"]):::drop + RETRY(["retriable ter → keep in set (no span)"]):::plain + + I1 --> FREE + I2 --> FREE + I3 --> REPF --> FREE + I4 --> FREE + FREE --> PF + PF -->|preflight tesSUCCESS| PC + PF -. "else → classify (no preclaim/transactor)" .-> CLS + PC -->|likelyToClaimFee| TR + PC -. "else → classify (no transactor)" .-> CLS + TR --> CLS + CLS --> OK + CLS --> FAIL + CLS --> RETRY + RETRY -. "next pass while pass<3 and changes>0" .-> FREE + RETRY -. "last pass → drop from set" .-> FAIL +``` + +Pipeline gates and retry (evidence): + +- `preclaim` short-circuits if preflight `!tesSUCCESS` + ([applySteps.cpp:498](../src/libxrpl/tx/applySteps.cpp#L498)); `doApply` + short-circuits if `!likelyToClaimFee` + ([applySteps.cpp:532](../src/libxrpl/tx/applySteps.cpp#L532)); the transactor + mutates only when preclaim is `tesSUCCESS` + ([Transactor.cpp:1647](../src/libxrpl/tx/Transactor.cpp#L1647)). +- **Final-TER classification**: `applied` → Success; `tef | tem | tel` → hard Fail; + else → Retry ([apply.cpp:226](../src/libxrpl/tx/apply.cpp#L226)). +- **Multi-pass retry**: both open-ledger `applyOne` and consensus `tx.apply` loop + `pass < LEDGER_TOTAL_PASSES` (= 3); a `Retry` tx is kept for the next pass, and + the final pass converts lingering retriable txs into drops + ([OpenLedger.h:237](../src/xrpld/app/ledger/OpenLedger.h#L237), + [BuildLedger.cpp:129](../src/xrpld/app/ledger/detail/BuildLedger.cpp#L129); + `LEDGER_TOTAL_PASSES` [OpenLedger.h:29](../src/xrpld/app/ledger/OpenLedger.h#L29)). +- **TxQ re-preflight**: the queue path re-runs `preflight` when the ledger's + rules/flags changed since enqueue + ([TxQ.cpp:315](../src/xrpld/app/misc/detail/TxQ.cpp#L315)). +- **TxQ cross-ledger retry**: a queued tx that fails with a retriable result keeps its slot with + `--retriesRemaining` (`kRetriesAllowed` = 10) and is re-applied at a **later** + ledger close; on `retriesRemaining ≤ 0` or `tef|tem` it is dropped with an + account `retryPenalty` ([TxQ.cpp:1528](../src/xrpld/app/misc/detail/TxQ.cpp#L1528)). +- `TxQ::apply` outcome fork: preflight-reject / `applied_direct` / `batch_clear` / + `queued` (`terQUEUED`) / reject + ([TxQ.cpp:762](../src/xrpld/app/misc/detail/TxQ.cpp#L762)). + +> **`tx.apply` is set-level, consensus-only.** It wraps the retry-pass loop over +> the agreed set during `buildLedger` and exists on **no other** invoker. It is +> not a per-transaction span, and TxQ / open-ledger apply create no `tx.apply`. + +### Consensus round + +`beginConsensus → startRound` starts the round (`consensus.round`, `Open` phase). +The **heartbeat timer** drives `Consensus::timerEntry` each pass; the round stays +in `Establish` across many heartbeats until the outcome is decided. + +> **A single ledger can take many rounds to settle.** Two nested multi-round +> mechanisms (see [docs/consensus.md](consensus.md)): +> +> 1. **Avalanche rounds inside one Establish phase** — each `timerEntry` runs +> `phaseEstablish` again (`establishCounter_++`) and raises the inclusion +> threshold **50% → 65% → 70% → 95%** as the round ages +> ([ConsensusParms.h:145](../src/xrpld/consensus/ConsensusParms.h#L145)). +> `checkConsensus` returning `No` keeps the node in `Establish` and loops; a +> round cannot even `Expire` before a minimum of +> `avalancheCutoffs.size() × avMinRounds = 4 × 2 = 8` passes +> ([Consensus.h:1938](../src/xrpld/consensus/Consensus.h#L1938)). +> 2. **Retry across consensus rounds** — a round can end `MovedOn` / `Expired`, +> meaning the network settled a _different_ ledger. The node still builds a +> ledger, but the **next** round's `checkLedger` detects the wrong prior, +> switches to `WrongLedger` / `SwitchedLedger` mode, acquires the correct +> ledger, and re-deliberates. A ledger is only truly settled once trusted +> **validations reach quorum** (`ledger.validate`); the alternate is abandoned. + +```mermaid +flowchart TB + classDef span fill:#b45309,stroke:#7c2d12,color:#fff; + classDef plain fill:#334155,stroke:#0f172a,color:#fff; + + BEGIN(["beginConsensus → startRound
(Proposing OR Observing; no span)"]):::plain + ROUND["consensus.round
(one attempt at next ledger)"]:::span + OPENS["consensus.phase.open
(collect txs; buffer peer
proposals / gotTxSet)"]:::span + HB(["heartbeat → timerEntry
(every LEDGER_MIN_CLOSE; no span)"]):::plain + CKL(["checkLedger
(correct prior ledger? no span)"]):::plain + WRONG(["handleWrongLedger → leaveConsensus (no span):
if Proposing send BOW-OUT,
mode → Observing; acquire ledger"]):::plain + MODE["consensus.mode_change
(mode transition)"]:::span + POPEN(["phaseOpen: shouldCloseLedger? (no span)"]):::plain + CLOSE["consensus.ledger_close
(close open ledger, seed disputes)"]:::span + pSend["consensus.proposal.send
(broadcast our position)"]:::span + PEST["consensus.establish
(phaseEstablish; avalanche round ↻
threshold 50→65→70→95%)"]:::span + UPOS["consensus.update_positions
(add/drop disputed txs; child of establish)"]:::span + acqTx(["acquireTxSet → gotTxSet
(async peer tx set; no span)"]):::plain + PAUSE(["shouldPause?
(wait on laggards; no span)"]):::plain + CHECK["consensus.check
(checkConsensus; child of establish)"]:::span + CTC(["haveCloseTimeConsensus?
(else agree-to-disagree +1s; no span)"]):::plain + ACCEPT["consensus.accept
(round complete)"]:::span + + BEGIN --> ROUND --> OPENS + HB -->|under mutex| CKL + CKL -.->|wrong prior| WRONG + WRONG --> MODE + WRONG -. "recovered → re-enter Open (playbackProposals)" .-> OPENS + WRONG -. "still missing → keep deliberating, defer to peers" .-> HB + CKL -->|prior OK| POPEN + HB -.->|phase==Open| POPEN + HB -.->|phase==Establish| PEST + POPEN -.->|shouldClose| CLOSE + CLOSE -.->|mode==Proposing| pSend + PEST --> UPOS + UPOS -.->|position changed && Proposing| pSend + UPOS -.->|disagreeing peer position| acqTx + acqTx -. "gotTxSet ↻ → new disputes" .-> UPOS + PEST --> PAUSE + PAUSE -. "pausing → wait (loop)" .-> HB + PAUSE -->|ready| CHECK + CHECK -. "No / Expired < 8 passes → next avalanche round" .-> HB + CHECK --> CTC + CTC -. "no CT consensus → loop" .-> HB + CTC -.->|Yes / MovedOn / Expired ≥ 8| ACCEPT + ROUND -.->|mode set at start| MODE + ACCEPT -. "endConsensus → next round ↻ (until a ledger validates)" .-> BEGIN +``` + +Consensus loops and branches (evidence): + +- **`consensus.establish` is the parent of `update_positions` and `check`**: + `phaseEstablish` creates the establish span (`startEstablishTracing`), and both + child spans parent to its captured context + ([Consensus.h:2100](../src/xrpld/consensus/Consensus.h#L2100), + [1629](../src/xrpld/consensus/Consensus.h#L1629), + [1838](../src/xrpld/consensus/Consensus.h#L1838)). +- **Avalanche-convergence loop (rounds within one ledger)**: repeated + `heartbeat → timerEntry → phaseEstablish` bumps `establishCounter_` and raises + the inclusion threshold each pass; `checkConsensus` = `No` stays in `Establish` + ([NetworkOPs.cpp:1214](../src/xrpld/app/misc/NetworkOPs.cpp#L1214); + [Consensus.h:1468](../src/xrpld/consensus/Consensus.h#L1468); + thresholds [ConsensusParms.h:145](../src/xrpld/consensus/ConsensusParms.h#L145)). +- **Retry-across-rounds loop (many rounds per settled ledger)**: `MovedOn` / + `Expired` accepts a non-preferred ledger; the next round's `checkLedger` finds + the wrong prior and recovers before re-deliberating + ([Consensus.h:1194](../src/xrpld/consensus/Consensus.h#L1194)); round-to-round + via `endConsensus → beginConsensus` + ([NetworkOPs.cpp:2315](../src/xrpld/app/misc/NetworkOPs.cpp#L2315)). +- **Two extra establish loop-backs before accept**: `shouldPause` (laggard + backpressure) and `!haveCloseTimeConsensus_` (TX consensus but not close-time) + each `return` and re-loop, distinct from `checkConsensus == No` + ([Consensus.h:1497](../src/xrpld/consensus/Consensus.h#L1497), + [1500](../src/xrpld/consensus/Consensus.h#L1500)); close time can + "agree to disagree" at prior close + 1s ([docs/consensus.md:163](consensus.md)). +- **acquireTxSet / gotTxSet loop**: a disagreeing peer position triggers an async + `acquireTxSet`; the later `gotTxSet` regenerates disputes and can extend the + establish phase ([Consensus.h:932](../src/xrpld/consensus/Consensus.h#L932)). +- **Bow-out / mode change**: `handleWrongLedger → leaveConsensus` sends a bow-out + proposal and demotes Proposing → Observing for the rest of the round + ([Consensus.h:1977](../src/xrpld/consensus/Consensus.h#L1977)); `startRound` + begins in Proposing **or** Observing ([docs/consensus.md:176](consensus.md)). +- **Buffered Open-phase inputs**: `peerProposal` / `gotTxSet` arriving during Open + are stored, then seeded as disputes at `closeLedger` (`createDisputes`); + `playbackProposals` replays them at `startRound` / `handleWrongLedger` + ([docs/consensus.md:244](consensus.md); + [Consensus.h:817](../src/xrpld/consensus/Consensus.h#L817)). +- **Outcome fork** after `checkConsensus`: `No` (loop) / `Yes` (onAccept) / + `MovedOn` / `Expired` ([Consensus.h:1516](../src/xrpld/consensus/Consensus.h#L1516)). +- **Expired guard**: a round cannot leave on `Expired` before + `avalancheCutoffs.size() × avMinRounds` (= 8) passes — below that, `Expired` + loops like `No` ([Consensus.h:1938](../src/xrpld/consensus/Consensus.h#L1938)). +- The **deterministic-vs-random trace-strategy** branch at round start + ([RCLConsensus.cpp:1291](../src/xrpld/app/consensus/RCLConsensus.cpp#L1291)) sets + only the trace ID — it has **zero protocol effect**. + +### Accept, build, and finalize the ledger + +`onAccept` enqueues a `JtAccept` job; `doAccept` runs on that worker +(`consensus.accept.apply`). It builds the ledger (running the apply pipeline over +the agreed set), cleans the queue, stores the ledger, optionally broadcasts a +validation, and rebuilds the open ledger. A built ledger is **not final** — it is +promoted to `ledger.validate` only when trusted validations reach quorum, an +**async, validation-driven** path re-entered per incoming trusted validation; a +built ledger that loses is **abandoned**. + +```mermaid +flowchart TB + classDef span fill:#6d28d9,stroke:#4c1d95,color:#fff; + classDef plain fill:#334155,stroke:#0f172a,color:#fff; + classDef drop fill:#7f1d1d,stroke:#ef4444,color:#fff; + + onAcc["consensus.accept
(round complete)"]:::span + APPLY["consensus.accept.apply
(JtAccept worker)"]:::span + BLCL(["buildLCL: replay data? (no span)"]):::plain + BUILD["ledger.build
(normal: apply agreed set)"]:::span + RPLY["ledger.build
(replay: TapNone, no tx.apply child)"]:::span + TXAP["tx.apply
(↻ each tx × up to 3 passes)"]:::span + CLEAN["txq.cleanup
(expire queue entries)"]:::span + STORE["ledger.store
(built, NOT yet final)"]:::span + vSend["consensus.validation.send
(broadcast our validation)"]:::span + CACC(["consensusBuilt → checkAccept
(quorum gate; no span)"]):::plain + NEWVAL(["inbound trusted validation
→ handleNewValidation → checkAccept
(async, per validation; no span)"]):::plain + VAL["ledger.validate
(promote highest-seq ledger ≥ quorum)"]:::span + LOSE(["built ledger loses:
never promoted → abandoned"]):::drop + OACC(["OpenLedger::accept
(rebuild open ledger; no span)"]):::plain + tqAcc["txq.accept
(↻ drain queued txs)"]:::span + swlStd(["switchLCL standalone:
setFullLedger + tryAdvance (no span)"]):::plain + swlNet(["switchLCL networked:
checkAccept (no span)"]):::plain + END(["endConsensus → next round ↻ (no span)"]):::plain + + onAcc -.->|addJob JtAccept| APPLY + APPLY --> BLCL + BLCL -->|normal path| BUILD --> TXAP + BLCL -. "replay path" .-> RPLY + APPLY --> CLEAN + APPLY --> STORE + APPLY -. "validating && isCompatible && !fail && canValidateSeq" .-> vSend + APPLY --> CACC + NEWVAL --> CACC + CACC -.->|highest-seq trusted ledger ≥ quorum| VAL + CACC -.->|tvc < quorum → no promotion| LOSE + APPLY --> OACC + OACC -->|TxQ::accept callback| tqAcc + OACC --> swlStd + OACC --> swlNet + swlStd -. "marks full-validated (no ledger.validate span)" .-> END + swlNet --> CACC + onAcc --> END +``` + +- Order inside `doAccept`: `buildLCL` (build → `tx.apply`, then `txq.cleanup`, + then `ledger.store`) → optional `validate` → `consensusBuilt`/`checkAccept` → + `OpenLedger::accept` (rebuilds the open ledger; `txq.accept` runs in its + callback) → `switchLCL` promotes the built ledger to the new LCL + ([RCLConsensus.cpp:812](../src/xrpld/app/consensus/RCLConsensus.cpp#L812) then + [833](../src/xrpld/app/consensus/RCLConsensus.cpp#L833)). +- **buildLCL replay branch**: if `releaseReplay()` has data, `buildLedger` replays + the stored set with `TapNone` — it **still emits `ledger.build`** (via + `buildLedgerImpl`) but applies txns directly with **no `tx.apply` child** and no + 3-pass loop; else the normal consensus-set path runs `tx.apply` over 3 passes + ([RCLConsensus.cpp:929](../src/xrpld/app/consensus/RCLConsensus.cpp#L929); + [BuildLedger.cpp:252](../src/xrpld/app/ledger/detail/BuildLedger.cpp#L252)). +- `processClosedLedger` (`txq.cleanup`) runs **after** build, **before** store + ([RCLConsensus.cpp:950](../src/xrpld/app/consensus/RCLConsensus.cpp#L950) vs + [953](../src/xrpld/app/consensus/RCLConsensus.cpp#L953)). +- **`ledger.validate` is async + lossy**: `checkAccept` is re-entered per incoming + trusted validation (`handleNewValidation → checkAccept`, + [RCLValidations.cpp:193](../src/xrpld/app/consensus/RCLValidations.cpp#L193)); + it promotes the **highest-seq** trusted ledger whose `valCount > neededValidations` + ([LedgerMaster.cpp:1180](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L1180)), + which may be a **different** ledger than the one this node built. Below quorum + (`tvc < minVal`) it returns early with no promotion — a built ledger that loses + is abandoned ([LedgerMaster.cpp:980](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L980); + [docs/consensus.md:50](consensus.md)). The `ledger.validate` span is emitted only + inside `checkAccept` ([LedgerMaster.cpp:987](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L987)). +- **validation-send guard**: broadcast only if + `validating_ && isCompatible && !consensusFail && canValidateSeq(seq)` — silently + suppressed for incompatible ledgers or an already-validated seq + ([RCLConsensus.cpp:730](../src/xrpld/app/consensus/RCLConsensus.cpp#L730)). +- **switchLCL**: standalone → `setFullLedger` + `tryAdvance` — marks the ledger + full-validated **without** emitting `ledger.validate` (that span lives only in + `checkAccept`); networked → `checkAccept` (shared async quorum gate) + ([LedgerMaster.cpp:442](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L442)). + +### Side flows: pathfinding and ledger acquire + +**Pathfinding** — an RPC one-shot (`path_find` / `ripple_path_find`) plus an async +recompute that fires on **every ledger close** for all active subscriptions, and +also garbage-collects dead subscriptions: + +```mermaid +flowchart TB + classDef span fill:#0e7490,stroke:#155e75,color:#fff; + classDef plain fill:#334155,stroke:#0f172a,color:#fff; + classDef drop fill:#7f1d1d,stroke:#ef4444,color:#fff; + + REQ["pathfind.request
(path_find / ripple_path_find)"]:::span + CREATE(["subscribe: makePathRequest →
persistent subscription (no span)"]):::plain + COMP["pathfind.compute
(doUpdate, one pass)"]:::span + DISC["pathfind.discover
(findPaths)"]:::span + PFDR(["Pathfinder + RippleCalc
↻ per source asset (no span)"]):::plain + UP(["updatePaths (JtUpdatePf, every close; no span)"]):::plain + UALL["pathfind.update_all
(recompute all active)"]:::span + DEAD(["dead subscriber → doAborting +
remove_if erase"]):::drop + + REQ -.->|subcommand create| CREATE + REQ -->|doUpdate| COMP -->|findPaths| DISC --> PFDR + UP -->|once per close| UALL + UALL -->|↻ each active request| COMP + UALL -. "new request arrived → extra pass ↻" .-> UALL + UALL -.->|dead / aborted| DEAD +``` + +**Ledger acquire** — a **separate trace root** (not part of the close flow) that +fetches a missing or correct-prior ledger from peers, retries per peer/timer, and +finishes with a reason-dependent store; `checkAccept` + `tryAdvance` run on **any** +completed acquire: + +```mermaid +flowchart TB + classDef span fill:#0e7490,stroke:#155e75,color:#fff; + classDef plain fill:#334155,stroke:#0f172a,color:#fff; + classDef drop fill:#7f1d1d,stroke:#ef4444,color:#fff; + + HIST(["tryAdvance → doAdvance → fetchForHistory
(Reason::HISTORY; no span)"]):::plain + NEED(["checkAccept / handleNewValidation /
consensus wrong-ledger (no span)"]):::plain + INB(["InboundLedgers::acquire (no span)"]):::plain + ACQ["ledger.acquire
(InboundLedger::init)"]:::span + TRIG(["trigger / addPeers / onTimer
↻ per peer / chunk (no span)"]):::plain + FAILED(["timeouts > 6 → failed_ →
logFailure (NO store, NO checkAccept)"]):::drop + DONE(["done() — complete && !failed (no span)"]):::plain + ONF(["onLedgerFetched (no span)
(HISTORY: no store)"]):::plain + STORE["ledger.store
(GENERIC / CONSENSUS)"]:::span + CACC(["checkAccept + tryAdvance (no span)
(↻ may publish/advance many ledgers)"]):::plain + + NEED -.->|GENERIC / CONSENSUS| INB --> ACQ + HIST -.->|HISTORY| INB + ACQ -.->|not complete| TRIG + TRIG -. "retry ↻" .-> TRIG + TRIG -.->|timeout cap| FAILED + TRIG -.->|complete| DONE + DONE -.->|reason == HISTORY| ONF + DONE -.->|GENERIC / CONSENSUS| STORE + DONE --> CACC + CACC -. "advanceWork ↻ → further HISTORY acquire" .-> HIST +``` + +Side-flow evidence: + +- **Pathfind subscription lifecycle**: `path_find` create inserts a persistent + subscription (`makePathRequest`); `update_all` re-runs each active request every + close, removes dead subscribers (`doAborting` + `remove_if` erase), and takes an + extra pass when a new request arrived mid-run + ([PathRequestManager.cpp:103](../src/xrpld/rpc/detail/PathRequestManager.cpp#L103), + [169](../src/xrpld/rpc/detail/PathRequestManager.cpp#L169), + [181](../src/xrpld/rpc/detail/PathRequestManager.cpp#L181)). +- **Acquire outcome fork**: `timeouts_ > kLedgerTimeoutRetriesMax` (= 6) sets + `failed_` → terminal `logFailure`, no store/checkAccept + ([InboundLedger.cpp:387](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L387)). +- **done() reason branch (store side only)**: `HISTORY` → `onLedgerFetched`, **no** + `storeLedger`; else → `storeLedger`. But `checkAccept` + `tryAdvance` run for + **any** `complete_ && !failed_` acquire regardless of reason + ([InboundLedger.cpp:495](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L495) + store switch; [507](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L507) + reason-independent checkAccept/tryAdvance). +- **tryAdvance multi-ledger loop**: `doAdvance` runs `do { … } while (advanceWork_)`, + publishing a range of ledgers and recursively triggering further HISTORY acquire + ([LedgerMaster.cpp:1905](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L1905)). + +### Where telemetry parenting differs from protocol flow + +The graph above is protocol control flow. The OpenTelemetry span **parent links** +are built differently and, in several places, do **not** represent a real +call edge. Read a trace with these in mind: + +| Telemetry does this | Real protocol flow | +| ------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `tx.process` is a `hashSpan` root from `txID` — an independent trace root ([TxTracing.h:63](../src/xrpld/telemetry/TxTracing.h#L63)). | The real edge is the synchronous `doSubmit → processTransaction` call; it is **not** a child of `rpc.command.submit`. | +| `tx.preflight` / `tx.preclaim` / `tx.transactor` share one `txID`-derived trace ID. | That shared ID is a correlation trick, not a call edge. The real order is the composed `apply()` at [apply.cpp:118](../src/libxrpl/tx/apply.cpp#L118). They are **not** children of `tx.process` or `tx.apply`. | +| `consensus.round` uses a deterministic trace ID from the previous ledger hash. | This makes **all validators share one trace ID** (a cross-node shared root), not a per-node parent. The real round-to-round edge is `endConsensus → beginConsensus`. | +| `consensus.accept` (main thread) and `consensus.accept.apply` (JtAccept worker) are wired via a captured context. | The real edge is the queued `JtAccept` job, a thread hand-off ([RCLConsensus.cpp:483](../src/xrpld/app/consensus/RCLConsensus.cpp#L483)). | +| `pathfind.update_all` parents nothing from the original `pathfind.request`. | The causal link is the ledger-close job on `JtUpdatePf`, not span nesting. | +| `ledger.acquire` and its downstream `ledger.store` / `ledger.validate`. | Reached via the `AcqDone` job, not parent inheritance; `ledger.acquire` is its own root. | +| `peer.*.receive` (fresh `kConsumer` root) and `consensus.*.receive` on the same message. | Two **sequential stages of one synchronous handler**, not parent/child; on a duplicate/untrusted drop the `consensus.*.receive` is never created. | +| Receive spans adopt the sender's `trace_id` + `span_id` as a genuine cross-node parent. | Deliberate: the receive span becomes a child of a **different node's** span (a cross-node context marker, not an in-process edge). `tx.receive` is asymmetric — it borrows only the sender's `span_id` and re-derives its own `trace_id` from `txID`. | + +> **Known telemetry artifacts** (from live audits, memory `otel-span-hierarchy-audit`): +> an RPC entry span's scope can leak across a reused coroutine worker, and the +> `hashSpan` roots (`tx.*`) — along with plain roots like `ledger.acquire` — can +> surface in Tempo as dangling "root span not yet received". These are +> exporter/parenting artifacts, not real control-flow parents. + +--- + +## Insights and Sample Queries + +This section shows what questions you can answer using the span attributes, with example Tempo TraceQL queries. + +**TraceQL syntax note:** span attributes must be referenced with the `span.` prefix inside `{}`. +Conditions are combined with `&&`. The `|` pipeline operator is not supported on this Tempo version. + +``` +# General pattern +{name="" && span. = && span. != } + +# Duration filter (no prefix needed) +{name="" && duration > 500ms} + +# Regex match +{name="" && span. =~ ".*"} + +# Multiple span names +{name = "" || name = ""} + +# Name regex +{name =~ ".*" && span. = } + +# Structural: find parent spans that have a matching child/event +{name=""} >> {event:name=""} +``` + +### Transaction Workflow Analysis + +``` +# Find all AMM transactions (AMMDeposit, AMMWithdraw, AMMVote) +{name="tx.process" && span.tx_type =~ "AMM.*"} + +# Find a specific AMM operation +{name="tx.process" && span.tx_type = "AMMDeposit"} +{name="tx.process" && span.tx_type = "AMMWithdraw"} +{name="tx.process" && span.tx_type = "AMMVote"} + +# Find Payment transactions that failed +{name="tx.process" && span.tx_type = "Payment" && span.ter_result != "tesSUCCESS"} + +# Find Payment failures due to path issues +{name="tx.process" && span.tx_type = "Payment" && span.ter_result =~ "tecPATH.*"} + +# Compare latency of different transaction types +{name="tx.process" && span.tx_type = "OfferCreate"} +{name="tx.process" && span.tx_type = "Payment"} + +# Find high-fee transactions (fee > 1 XRP = 1000000 drops) +{name="tx.process" && span.fee > 1000000} + +# Find transactions that were not applied +{name="tx.process" && span.applied = false} + +# Find NFTokenMint across tx and txq spans +{name =~ "tx.*|txq.*" && span.tx_type = "NFTokenMint"} + +# Find all NFT-related activity +{name =~ "tx.*|txq.*" && span.tx_type =~ "NFToken.*"} + +# Find TrustSet transactions (IOU trust lines) +{name="tx.process" && span.tx_type = "TrustSet"} + +# Find oracle price updates +{name="tx.process" && span.tx_type = "OracleSet"} +``` + +### DEX (OfferCreate / OfferCancel) + +``` +# All DEX offer creates +{name="tx.process" && span.tx_type = "OfferCreate"} + +# Offers killed (ImmediateOrCancel/FillOrKill with no fill) +{name="tx.process" && span.tx_type = "OfferCreate" && span.ter_result = "tecKILLED"} + +# Offers that failed due to insufficient funds +{name="tx.process" && span.tx_type = "OfferCreate" && span.ter_result = "tecUNFUNDED_OFFER"} + +# Offers failed due to insufficient reserve to place the offer +{name="tx.process" && span.tx_type = "OfferCreate" && span.ter_result = "tecINSUF_RESERVE_OFFER"} + +# Offer cancellations +{name="tx.process" && span.tx_type = "OfferCancel"} + +# OfferCreate transactions received from peers (cross-node relay) +{name="tx.receive" && span.tx_type = "OfferCreate"} +``` + +### Apply Pipeline by Stage + +``` +# All three stages of one transaction (preflight -> preclaim -> apply) +{name=~"tx.preflight|tx.preclaim|tx.transactor"} + +# Transactions that failed at the preclaim stage +{name="tx.preclaim"} | ter_result != "tesSUCCESS" + +# Transactions that hard-failed preflight (never reached preclaim/apply) +{name="tx.preflight"} | ter_result != "tesSUCCESS" +``` + +PromQL on the span-derived metrics (dashboard: _Transaction Overview_): + +``` +# Per-stage throughput — the funnel preflight >= preclaim >= apply +sum by (stage) (rate(span_calls_total{span_name=~"tx.preflight|tx.preclaim|tx.transactor"}[5m])) + +# Per-stage p95 latency +histogram_quantile(0.95, sum by (le, stage) (rate(span_duration_milliseconds_bucket{span_name=~"tx.preflight|tx.preclaim|tx.transactor"}[5m]))) + +# Per-stage failure rate (ter_result != tesSUCCESS; a failing ter completes the +# span normally, so filter on the attribute, not status_code which only flags exceptions) +sum by (stage) (rate(span_calls_total{span_name=~"tx.preflight|tx.preclaim|tx.transactor", ter_result!~"tesSUCCESS|"}[5m])) +``` + +> **Alerting**: a rising `tx.preflight` / `tx.preclaim` failure rate points to +> malformed or stale-sequence submissions (often spam or a misbehaving client); +> a rising `tx.transactor` failure rate points to apply-time problems. Alert per +> stage rather than on a single aggregate so the failing stage is obvious. + +> **Sampling caveat**: these stage metrics are span-derived and inherit the +> **tracer head-sampling** ratio (`sampling_ratio`). At `sampling_ratio < 1.0` +> they undercount proportionally — treat them as relative trends, not absolute +> transaction counts. Native StatsD metrics are unsampled. + +### Transaction Queue Health + +``` +# Find transactions rejected from the queue +{name="txq.accept_tx" && span.txq_status = "failed"} + +# Find transactions being retried +{name="txq.accept_tx" && span.txq_status = "retried"} + +# Find transactions that exhausted retries +{name="txq.accept_tx" && span.txq_status = "retried" && span.retries_remaining = 0} + +# Which transaction types get queued most often? +{name="txq.enqueue" && span.tx_type = "Payment"} +{name="txq.enqueue" && span.tx_type = "OfferCreate"} +{name="txq.enqueue" && span.tx_type =~ "NFToken.*"} + +# Find ledger closes that applied queued transactions +{name="txq.accept" && span.ledger_changed = true} +``` + +### RPC Debugging + +``` +# Find batch RPC requests +{name="rpc.process" && span.is_batch = true} + +# Find large RPC payloads (>100KB) +{name="rpc.http_request" && span.request_payload_size > 100000} + +# Find resource-heavy RPC commands (by load_type) +{name =~ "rpc.command.*" && span.load_type = "exceptioned RPC"} + +# Find a specific WebSocket command +{name="rpc.ws_message" && span.command = "subscribe"} + +# Find server_info calls +{name="rpc.command.server_info"} + +# Find slow pathfinding with many source assets +{name="pathfind.discover" && span.pathfind_num_source_assets > 10} +``` + +### PathFinding Performance + +``` +# Find pathfinding for specific currencies +{name="pathfind.compute" && span.pathfind_dest_currency = "USD"} + +# Find expensive pathfinding (many source assets to explore) +{name="pathfind.discover" && span.pathfind_num_source_assets > 20} + +# Find slow pathfinding requests +{name="pathfind.compute" && duration > 1000ms} +``` + +### Consensus Health + +``` +# Find rounds where consensus timed out (expired) +{name="consensus.accept" && span.consensus_state = "expired"} + +# Find rounds where we moved on without full agreement +{name="consensus.accept" && span.consensus_state = "moved_on"} + +# Find rounds with many disputes +{name="consensus.accept" && span.disputes_count > 5} + +# Find slow consensus rounds (>5s) +{name="consensus.accept" && span.round_time_ms > 5000} + +# Find bow-out proposals (node resigned from round) +{name="consensus.proposal.send" && span.is_bow_out = true} + +# Correlate validation with its ledger +{name="consensus.validation.send" && span.ledger_hash = ""} + +# Find rounds where validators disagreed on close time +{name="consensus.accept.apply" && span.close_time_correct = false} + +# Find both validation send and receive (compare sender vs receiver latency) +{name = "consensus.validation.send" || name = "consensus.validation.receive"} +``` + +### Cross-Subsystem Correlation + +``` +# Follow a transaction from receive through queue to ledger +{name =~ "tx.*|txq.*" && span.tx_type = "Payment" && duration > 500ms} + +# Find all NFT-related activity across tx and txq spans +{name =~ "tx.*|txq.*" && span.tx_type =~ "NFToken.*"} + +# Find all AMM activity across tx and txq spans +{name =~ "tx.*|txq.*" && span.tx_type =~ "AMM.*"} + +# Find cross-node transaction receives (no errors) +{name="tx.receive" && status != error} +``` + +### Where to Look (Quick Reference) + +| Question | Span | Key Attributes | +| ----------------------------------- | --------------------------- | ---------------------------------------- | +| "Which tx type is slowest?" | `tx.process` | `span.tx_type` + duration | +| "Why was my tx rejected?" | `tx.process` | `span.ter_result`, `span.applied` | +| "What AMM operations happened?" | `tx.process` | `span.tx_type =~ "AMM.*"` | +| "What DEX offers failed?" | `tx.process` | `span.tx_type`, `span.ter_result` | +| "What NFT activity occurred?" | `tx.process`, `txq.enqueue` | `span.tx_type =~ "NFToken.*"` | +| "Is the TxQ backing up?" | `txq.accept` | `span.queue_size`, `span.ledger_changed` | +| "Why was my tx dropped from queue?" | `txq.accept_tx` | `span.txq_status`, `span.ter_code` | +| "Are batch requests a problem?" | `rpc.process` | `span.is_batch`, `span.batch_size` | +| "Which RPC is expensive?" | `rpc.command.*` | `span.load_type`, duration | +| "Did consensus reach threshold?" | `consensus.check` | `span.consensus_result` | +| "Was consensus outcome normal?" | `consensus.accept` | `span.consensus_state` | +| "Did a validator bow out?" | `consensus.proposal.send` | `span.is_bow_out` | +| "Which ledger was validated?" | `consensus.validation.send` | `span.ledger_hash` | +| "Did close time agreement fail?" | `consensus.accept.apply` | `span.close_time_correct` | +| "What tx work fed a given ledger?" | `tx.*` / `txq.*` | `span.current_ledger_seq` | + +### Correlating a transaction to the ledger it was worked on + +The `tx.process`, `tx.receive`, `txq.enqueue`, `tx.preclaim`, and `tx.transactor` +spans carry `current_ledger_seq` — the open/in-flight ledger they acted on (not +an established ledger). Because these spans are keyed on the transaction id (their +own trace) while the ledger/consensus spans are keyed on the ledger, use the +attribute to bridge the two id-spaces: + +``` +# All transaction-side work recorded against ledger N +{span.current_ledger_seq = N} + +# Join to the ledger build/consensus trace for the same ledger +{name="ledger.build" && span.ledger_seq = N} +``` + +`txq.enqueue` and the view-bearing apply stages also carry `current_ledger_hash` +(the current ledger's parent hash), which equals the `consensus.round` +deterministic trace-id seed on the consensus-build path. `tx.preflight` is +stateless and omits both attributes. + +--- + +## Cross-Node Trace Propagation + +xrpld propagates trace context across nodes via protobuf `TraceContext` fields +embedded in peer-to-peer messages. When Node A sends a transaction, proposal, +or validation, it injects its active span's trace/span IDs into the protobuf +message. Node B extracts that context on receipt and creates a child span, +linking the two nodes into a single distributed trace. + +### How It Works + +``` +Node A (sender) Node B (receiver) ++-----------------------------+ +-------------------------------+ +| tx.process / consensus.* | | PeerImp::onMessage() | +| | | | | | +| v | | v | +| SpanGuard::getTraceBytes() | | extract TraceContext from | +| | | | protobuf message | +| v | send | | | +| injectSpanContext() --------|--------->| v | +| sets TraceContext fields | proto | txReceiveSpan() | +| (trace_id, span_id, flags) | msg | proposalReceiveSpan() | ++-----------------------------+ | validationReceiveSpan() | + | | | + | v | + | child span with parent link | + +-------------------------------+ +``` + +### Send-Side Injection + +| Message Type | Injection Point | Mechanism | +| ------------- | -------------------------- | ------------------------------------------ | +| TMTransaction | `NetworkOPs::apply()` | Injects `tx.process` span into relay msg | +| TMProposeSet | `RCLConsensus::propose()` | Injects active context into proposal msg | +| TMValidation | `RCLConsensus::validate()` | Injects active context into validation msg | + +### Receive-Side Extraction + +| Message Type | Extraction Point | Helper Function | +| ------------- | ----------------------------------- | -------------------------------------------------- | +| TMTransaction | `PeerImp::onMessage(TMTransaction)` | `TxTracing::txReceiveSpan()` | +| TMProposeSet | `PeerImp::onMessage(TMProposeSet)` | `ConsensusReceiveTracing::proposalReceiveSpan()` | +| TMValidation | `PeerImp::onMessage(TMValidation)` | `ConsensusReceiveTracing::validationReceiveSpan()` | + +### Key Files + +| File | Role | +| ------------------------------------------------- | ----------------------------------------------- | +| `src/xrpld/telemetry/PropagationHelpers.h` | `injectSpanContext()` — SpanGuard to protobuf | +| `include/xrpl/telemetry/TraceContextPropagator.h` | OTel context <-> protobuf conversion primitives | +| `src/xrpld/telemetry/ConsensusReceiveTracing.h` | Proposal/validation receive span factories | +| `src/xrpld/telemetry/TxTracing.h` | Transaction receive span factory | + +### Backwards Compatibility + +Older peers that do not populate `TraceContext` fields in their messages will +simply produce empty trace bytes on the receive side. The extraction helpers +detect this and create standalone (root) spans instead of child spans. No +errors are logged and no data is lost — the receive span is still created with +all its normal attributes, it just lacks a cross-node parent link. + +### Example Tempo Queries + +``` +# Find cross-node transaction traces (tx.receive spans with no errors) +{name="tx.receive" && status != error} + +# Find proposals received with cross-node parent context +{} >> {name="consensus.proposal.receive"} + +# Trace a transaction across the network by its hash +{name =~ "tx.*" && span.tx_hash = ""} + +# Find all spans in a cross-node consensus trace +{resource.service.name="xrpld" && span.consensus_round_id = ""} + +# Compare latency between sender and receiver for validations +{name = "consensus.validation.send" || name = "consensus.validation.receive"} +``` + +## Prometheus Metrics (Spanmetrics) + +The OTel Collector's spanmetrics connector automatically derives RED (Rate, Errors, Duration) metrics from every span. No custom metrics code is needed in xrpld. + +### Generated Metric Names + +| Prometheus Metric | Type | Description | +| ----------------------------------- | --------- | ---------------------------- | +| `span_calls_total` | Counter | Total span invocations | +| `span_duration_milliseconds_bucket` | Histogram | Latency distribution buckets | +| `span_duration_milliseconds_count` | Histogram | Latency observation count | +| `span_duration_milliseconds_sum` | Histogram | Cumulative latency | + +### Metric Labels + +Every metric carries these standard labels: + +| Label | Source | Example | +| -------------- | ------------------ | ---------------------------------------- | +| `span_name` | Span name | `rpc.command.server_info` | +| `status_code` | Span status | `STATUS_CODE_UNSET`, `STATUS_CODE_ERROR` | +| `service_name` | Resource attribute | `xrpld` | +| `span_kind` | Span kind | `SPAN_KIND_INTERNAL` | + +Additionally, span attributes configured as dimensions in the collector +become metric labels. The span attribute keys are already underscore form +(the naming convention forbids dots), so the label name matches the attribute +name verbatim. Prometheus' dots → underscores sanitization only fires for +dotted attribute names (e.g. resource attributes like `service.name`), which +does not apply to these dimensions. + +| Span Attribute | Metric Label | Applies To | +| -------------------- | -------------------- | ------------------------------- | +| `command` | `command` | `rpc.command.*` spans | +| `rpc_status` | `rpc_status` | `rpc.command.*` spans | +| `consensus_mode` | `consensus_mode` | `consensus.ledger_close` spans | +| `local` | `local` | `tx.process` spans | +| `proposal_trusted` | `proposal_trusted` | `peer.proposal.receive` spans | +| `validation_trusted` | `validation_trusted` | `peer.validation.receive` spans | + +### Histogram Buckets + +Configured in `otel-collector-config.yaml` (spanmetrics connector, `unit: ms`): + +``` +1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2s, 3s, 4s, 5s, 10s, 30s +``` + +Sub-second boundaries cover RPC/tx/ledger spans; 2s-4s resolve second-scale +consensus spans (`consensus.round`, `consensus.establish`) that would otherwise +pile into one 1s-5s bucket and make `histogram_quantile` a meaningless +interpolation; 10s/30s give the `ledger.acquire` catch-up tail a measurable home. +Boundaries must stay strictly ascending. The native beast::insight histograms +(ms-scale RPC/IO timers) keep the original 1ms-5s buckets in +`Telemetry.cpp` — they never exceed 5s, so they need no high-range buckets. + +## System Metrics (OTel native -- beast::insight) + +xrpld has a built-in metrics framework (`beast::insight`) that exports metrics natively via OTLP to the OTel Collector. These complement the span-derived RED metrics by providing system-level gauges, counters, and timers that don't map to individual trace spans. + +### Configuration + +Add to `xrpld.cfg`: + +```ini +[insight] +server=otel +endpoint=http://localhost:4318/v1/metrics +prefix=xrpld +``` + +The `OTelCollector` implementation exports metrics via OTLP/HTTP to the same OTel Collector that receives traces. No separate StatsD receiver is needed. + +> **Fallback**: Set `server=statsd` and `address=127.0.0.1:8125` to use the legacy StatsD UDP path. This requires re-enabling the `statsd` receiver in `otel-collector-config.yaml` and uncommenting port 8125 in `docker-compose.yml`. + +### Metric Reference + +#### Gauges + +| Prometheus Metric | Source | Description | +| ------------------------------------- | ------------------------- | -------------------------------------------------------------------------- | +| `ledgermaster_validated_ledger_age` | LedgerMaster.h:373 | Age of validated ledger (seconds) | +| `ledgermaster_published_ledger_age` | LedgerMaster.h:374 | Age of published ledger (seconds) | +| `state_accounting_{mode}_duration` | NetworkOPs.cpp:774 | Time in each operating mode (Disconnected/Connected/Syncing/Tracking/Full) | +| `state_accounting_{mode}_transitions` | NetworkOPs.cpp:780 | Transition count per mode | +| `peer_finder_active_inbound_peers` | PeerfinderManager.cpp:214 | Active inbound peer connections | +| `peer_finder_active_outbound_peers` | PeerfinderManager.cpp:215 | Active outbound peer connections | +| `overlay_peer_disconnects` | OverlayImpl.h:557 | Peer disconnect count | +| `jobq_job_count` | JobQueue.cpp:26 | Current job queue depth (all types) | +| `jobq_{jobtype}_waiting` | JobTypeData.h | Jobs of this type enqueued but not yet running | +| `jobq_{jobtype}_running` | JobTypeData.h | Jobs of this type currently executing | +| `jobq_{jobtype}_deferred` | JobTypeData.h | Jobs of this type held back because the type's concurrency limit was hit | +| `{category}_bytes_in/out` | OverlayImpl.h:535 | Overlay traffic bytes per category (57 categories) | +| `{category}_messages_in/out` | OverlayImpl.h:535 | Overlay traffic messages per category | + +Note that `job_count` is exported as `jobq_job_count`: the JobQueue is +constructed with `collectorManager_->group("jobq")` (Application.cpp:386), +`GroupImp::makeName()` joins prefix and name with a `.` (Groups.cpp:42), and +`OTelCollectorImp::formatName()` then turns the `.` into `_` and lowercases the +whole string (OTelCollector.cpp:860). The same mechanism produces the +`jobq_{jobtype}_*` names above and the pre-existing +`jobq_{jobtype}_milliseconds` timing family. + +#### Per-Job-Type Queue Saturation + +The three `jobq_{jobtype}_{waiting,running,deferred}` families expose the +per-type counters that `JobTypeData` already maintained but never exported. +`{jobtype}` is the lowercased `JobTypes` name, so `JtLedgerReq` ("ledgerRequest") +becomes `jobq_ledgerrequest_waiting` / `_running` / `_deferred`. + +They are emitted for every **non-special** job type — 35 of the 46 declared +types. A "special" type is one whose concurrency limit is 0 +(`JobTypeInfo::special()`, JobTypeInfo.h:71-74); the limit logic never applies +to it, so its `deferred` is always zero. The gauge members are declared at +JobTypeData.h:78-80 and created at :100-102, next to the existing +`dequeue`/`execute` events and under the same `!info.special()` guard (:95). They +are published by `JobQueue::collect()` under the `mutex_` that already guards the +counters (JobQueue.cpp:66-93) — no new locking. + +**`deferred` is the leading indicator.** `JobQueue::addJob()` never rejects +work — when a type is at its limit, `addRefCountedJob()` increments `deferred` +and returns `true` anyway (JobQueue.cpp:131-142), and `finishJob()` drains one +deferred job per completion (JobQueue.cpp:353-362). Backpressure on a capped type +therefore shows up **only as latency**, after the harm is done. `deferred > 0` +says the cap is being hit _now_, before the duration histograms move. + +Limits that matter for ledger sync (JobTypes.h:54-77): + +| Job type | Metric prefix | Limit | Producers | +| -------------- | --------------------- | ----- | ------------------------------------------------------------------------------- | +| `JtPack` | `jobq_makefetchpack_` | 1 | `MakeFetchPack` | +| `JtLedgerReq` | `jobq_ledgerrequest_` | 3 | `RcvGetLedger`, `RcvGetObjByHash` | +| `JtLedgerData` | `jobq_ledgerdata_` | 3 | `ProcessLData`, `GotStaleData`, `GotFetchPack`, `AcqDone`, `InboundLedger` | +| `JtUpdatePf` | `jobq_updatepaths_` | 1 | `PthFindNewReq`, `PthFindOBDB`, `PthFindNewLed`, `OB` — see the note below | +| `JtTxnData` | `jobq_fetchtxndata_` | 5 | `TxAcq`, `ComplAcquire`, `RcvPeerData` | + +> **`JtUpdatePf` has four producers, three of them individually visible.** All +> four run the same `updatePaths()` work but arrive under different names. Three +> come through `LedgerMaster::newPFWork()` (LedgerMaster.cpp:1545), which passes +> its caller's name straight to `addJob`: `PthFindNewReq` (:1512), +> `PthFindOBDB` (:1533), and `PthFindNewLed` (:1984). All three are all-letters, +> so each is its own `handler` series. The fourth is +> `"OB" + std::to_string(seq)` (OrderBookDBImpl.cpp:84), which contains digits +> and therefore folds to `handler="other"` — order-book rebuild traffic is the +> only one of the four that is not directly attributable. Do not read the whole +> type as invisible: three of its four producers are named. + +> **Sampling caveat**: these are gauges read by the `JobQueue::collect()` hook, +> which the beast::insight `PeriodicMetricReader` drives every 1 s +> (Telemetry.cpp:441). A `deferred` spike shorter than the sample interval can be +> missed entirely. Treat a non-zero reading as real saturation, but do not treat +> a zero reading as proof that no saturation occurred — cross-check +> `job_queued_us` for the same type. + +#### OTel MetricsRegistry Gauges + +These gauges are exported via the OTel Metrics SDK `PeriodicMetricReader` (10s interval), NOT through beast::insight. + +| Prometheus Metric | Source | Description | +| --------------------------------------------------- | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `server_info{metric="server_state"}` | MetricsRegistry.cpp | Operating mode (0=DISCONNECTED .. 4=FULL) | +| `server_info{metric="uptime"}` | MetricsRegistry.cpp | Seconds since server start | +| `server_info{metric="peers"}` | MetricsRegistry.cpp | Total connected peers | +| `server_info{metric="validated_ledger_seq"}` | MetricsRegistry.cpp | Validated ledger sequence number | +| `server_info{metric="ledger_current_index"}` | MetricsRegistry.cpp | Current open ledger sequence | +| `server_info{metric="peer_disconnects_resources"}` | MetricsRegistry.cpp | Cumulative resource-related peer disconnects | +| `server_info{metric="last_close_proposers"}` | MetricsRegistry.cpp | Proposers in last closed round | +| `server_info{metric="last_close_converge_time_ms"}` | MetricsRegistry.cpp | Last close convergence time (ms) | +| `server_info{metric="last_close_time"}` | MetricsRegistry.cpp | Network close time of last closed ledger (NetClock secs since XRPL epoch). Age = `time() - (value + 946684800)`; close interval = `1/rate(ledgers_closed_total)`, not a gauge delta | +| `build_info{version=""}` | MetricsRegistry.cpp | Info-style metric (always 1) | +| `complete_ledgers{bound="start\|end",index=""}` | MetricsRegistry.cpp | Complete ledger range start/end pairs | +| `db_metrics{metric="db_kb_total"}` | MetricsRegistry.cpp | Total database size (KB) | +| `db_metrics{metric="db_kb_ledger"}` | MetricsRegistry.cpp | Ledger database size (KB) | +| `db_metrics{metric="db_kb_transaction"}` | MetricsRegistry.cpp | Transaction database size (KB) | +| `db_metrics{metric="historical_perminute"}` | MetricsRegistry.cpp | Historical ledger fetches per minute | +| `cache_metrics{metric="AL_size"}` | MetricsRegistry.cpp | AcceptedLedger cache size | +| `nodestore_state{metric="node_reads_duration_us"}` | MetricsRegistry.cpp | Cumulative read time (microseconds) | +| `nodestore_state{metric="read_request_bundle"}` | MetricsRegistry.cpp | Read request bundle count | +| `nodestore_state{metric="read_threads_running"}` | MetricsRegistry.cpp | Active read threads | +| `nodestore_state{metric="read_threads_total"}` | MetricsRegistry.cpp | Total read threads configured | +| `rpc_in_flight_requests` | PerfLogImp.cpp | RPC requests currently executing (UpDownCounter) | + +#### Counters + +| Prometheus Metric | Source | Description | +| ------------------------- | --------------------- | ------------------------------ | +| `rpc_requests` | ServerHandler.cpp:108 | Total RPC request count | +| `ledger_fetches` | InboundLedgers.cpp:44 | Ledger fetch request count | +| `ledger_history_mismatch` | LedgerHistory.cpp:16 | Ledger hash mismatch count | +| `warn` | Logic.h:33 | Resource manager warning count | +| `drop` | Logic.h:34 | Resource manager drop count | + +#### Histograms + +| Prometheus Metric | Source | Description | +| ----------------- | --------------------- | ------------------------------ | +| `rpc_time` | ServerHandler.cpp:110 | RPC response time (ms) | +| `rpc_size` | ServerHandler.cpp:109 | RPC response size (bytes) | +| `ios_latency` | Application.cpp:438 | I/O service loop latency (ms) | +| `pathfind_fast` | PathRequests.h:23 | Fast pathfinding duration (ms) | +| `pathfind_full` | PathRequests.h:24 | Full pathfinding duration (ms) | + +#### Job Instruments + +These five come from the `PerfLog` job hooks, not from beast::insight, so they +are exported by the `MetricsRegistry` meter. `job_queued_us` and `job_running_us` +have explicit microsecond bucket views registered +(`addMicrosecondHistogramView()`, MetricsRegistry.cpp:253-254) spanning 100 µs to +60 s; without those the SDK default buckets stop at 10 ms and every quantile +saturates. + +| Prometheus Metric | Kind | Labels | Description | +| -------------------- | --------- | --------------------- | ------------------------------------ | +| `job_queued_total` | Counter | `job_type`, `handler` | Jobs enqueued | +| `job_started_total` | Counter | `job_type`, `handler` | Jobs dequeued and started | +| `job_finished_total` | Counter | `job_type`, `handler` | Jobs run to completion | +| `job_queued_us` | Histogram | `job_type`, `handler` | Time spent waiting in the queue (µs) | +| `job_running_us` | Histogram | `job_type`, `handler` | Time spent executing (µs) | + +#### The `handler` Label + +`job_type` names the queue a job ran on, not the code that submitted it. Several +job types have more than one producer, so `job_type` alone cannot attribute a +latency spike. The clearest case: `RcvGetLedger` (PeerImp.cpp:1566) and +`RcvGetObjByHash` (PeerImp.cpp:2603) both submit to `JtLedgerReq`, so both report +as `job_type="ledgerRequest"`. `JtLedgerData` has five producers, `JtUpdatePf` has +four, and `JtAdvance` has four. + +The `handler` label carries the name string passed to `addJob()` — the specific +call site. It resolves all of those at once, not just the GetObject path. + +**The value is sanitized, not raw.** A raw job name would be unbounded, because +two production job names embed a ledger sequence number: + +- `"Pub" + std::to_string(ledger->seq())` (LedgerPersistence.cpp:84) +- `"OB" + std::to_string(ledger->seq() % 1000000000)` (OrderBookDBImpl.cpp:84) + +A raw label would mint a new Prometheus series for every ledger — unbounded +growth at ~1 series every 3-5 s, forever. +`MetricsRegistry::sanitiseHandler()` (declared inline in +`src/xrpld/telemetry/MetricsRegistry.h`) therefore applies one rule: + +- Keep the name when it is **non-empty and every character is an ASCII letter**. +- Otherwise return the constant `"other"`. An empty name, a digit, a hyphen, or + any punctuation all fall here. + +Both dynamic names always contain digits, so both collapse to `"other"`. Because +the rule is a pure function of compile-time string literals, the label domain is +fixed at build time and cannot grow at runtime. That is a stronger guarantee than +an allowlist, which would silently mislabel any job added later; this rule +degrades to `"other"` instead. + +**Five static job names also fall into `"other"`.** Sweeping every `addJob` / +`addRefCountedJob` / `postCoro` / `newPFWork` / `TimeoutCounter::jobName` call +site outside `src/test` finds 48 distinct static name literals. 43 are +all-letters and pass through; five are not, and two more are built at runtime +from a ledger sequence. The domain is therefore **44 values** (43 names plus +`"other"`): + +| Name | Why it is not all-letters | Call site | +| ------------- | ------------------------- | ---------------------- | +| `GetConsL1` | digit | RCLConsensus.cpp:169 | +| `GetConsL2` | digit | RCLValidations.cpp:135 | +| `gRPC-Client` | hyphen | GRPCServer.cpp:156 | +| `RPC-Client` | hyphen | ServerHandler.cpp:332 | +| `WS-Client` | hyphen | ServerHandler.cpp:376 | + +> The consequence worth remembering: `handler="other"` is a **mixed bucket**, not +> a residual. It holds the two per-ledger dynamic names _and_ those five static +> ones, so `GetConsL1` and `GetConsL2` — two different `JtAdvance` producers — +> are not separable, and neither are the three RPC client-session names. Do not +> read a handler breakdown as exhaustive. The GetObject path is unaffected: +> `RcvGetObjByHash` and `RcvGetLedger` are all-letters and pass through as +> distinct series. +> +> The count is "at the time of writing" — it is a property of the source, not of +> the rule. Re-derive it from the call sites after adding a job rather than +> trusting this number. + +#### GetObject Request Metrics + +Five instruments on the `TMGetObjectByHash` query path, recorded via the +`XRPL_METRIC_*` macros at their call sites in `PeerImp.cpp`. Label cardinality is +fixed and tiny: two `result` values, two `reason` values. + +| Prometheus Metric | Kind | Labels | Description | +| --------------------------- | --------- | ----------------------------------------------- | ----------------------------------------------------- | +| `getobject_lookup_us` | Histogram | none | Time inside the NodeStore fetch loop (µs) | +| `getobject_request_objects` | Histogram | none | Objects requested per request | +| `getobject_lookups_total` | Counter | `result` = `hit` \| `miss` | NodeStore hit/miss volume | +| `getobject_rejected_total` | Counter | `reason` = `oversize` \| `malformed_ledgerhash` | Requests refused before any NodeStore access | +| `getobject_charge` | Histogram | none | Dynamic component of the differential resource charge | + +Aggregation choices worth knowing when reading these: + +- `getobject_lookup_us` times the **whole fetch loop once** + (`processGetObjectByHash()`, PeerImp.cpp:2713-2742 — the iteration cap is set + at :2713 and the loop ends at :2742), not each iteration. The loop can run up + to `kHardMaxReplyNodes` = 12288 times (Tuning.h:30); timing each + `fetchNodeObject()` would cost more than the lookups. It needs an + `addMicrosecondHistogramView()` entry for the same reason the job histograms + do — a 12288-lookup loop routinely exceeds 10 ms, so without the view the + metric saturates exactly when it matters. +- `getobject_lookups_total` is incremented **once per request with the batch + totals**, not once per object. A 12288-iteration loop incrementing per object + would be a measurable hot-path cost for no extra information. +- `getobject_request_objects` records `packet.objects_size()` — the _requested_ + count, which is what the charge bands price on, not the count actually found. +- `getobject_charge` records only the **dynamic** part returned by + `computeGetObjectByHashFee()` (PeerImp.cpp:3658-3681), applied at + PeerImp.cpp:2757-2758 just after the loop. The admission-time base charge is a + constant (`kFeeModerateBurdenPeer`, PeerImp.cpp:2634) and is already implied. +- `getobject_rejected_total` counts the two early returns in + `onMessage(TMGetObjectByHash)`: the malformed-ledgerhash check (PeerImp.cpp:2569, + counter at :2573) and the oversize gate (PeerImp.cpp:2585, counter at :2591). + Both fire before the job is enqueued, so a rejected request contributes to no + other GetObject metric. + +> On a healthy local network `getobject_rejected_total` reads zero — no honest +> peer sends an oversized request. Verify its panel with a synthetic oversized +> request; do not assume it works because the query parses. + +#### Adding a New Metric + + + + +Use the call-site macros in `src/xrpld/telemetry/MetricMacros.h` -- no +`MetricsRegistry.h`/`.cpp` edit is needed for any of these: + +| Need | Macro | +| ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Monotonic tally (never decreases) | `XRPL_METRIC_COUNTER_INC` / `_ADD` [+ `_LABELED`] | +| Running total that can decrease | `XRPL_METRIC_UPDOWN_ADD` [+ `_LABELED`] | +| Distribution of values (latency, size) | `XRPL_METRIC_HISTOGRAM_RECORD` [+ `_LABELED`] | +| Last-value snapshot (not a distribution) | `XRPL_METRIC_GAUGE_RECORD` [+ `_LABELED`] -- requires an ABI v2 opentelemetry-cpp build; this repo currently builds ABI v1, so use the observable-gauge row below instead | +| Value your own code already tracks, sampled on a timer | `XRPL_METRIC_OBSERVABLE_GAUGE_REGISTER` / `_COUNTER_REGISTER` / `_UPDOWN_REGISTER` | + +```cpp +#include + +// Monotonic counter: +XRPL_METRIC_COUNTER_INC(app_, "my_new_thing_total", "Description of what this counts"); + +// Value that can go up and down, e.g. in-flight work (no _total suffix -- that +// is reserved for monotonic counters; an UpDownCounter is a current value): +XRPL_METRIC_UPDOWN_ADD(app_, "my_in_flight_requests", "Currently executing", 1); // on start +XRPL_METRIC_UPDOWN_ADD(app_, "my_in_flight_requests", "Currently executing", -1); // on finish + +// Sampled from your own state, on the OTel export timer (register ONCE, in init code): +XRPL_METRIC_OBSERVABLE_GAUGE_REGISTER(app_, "my_thing_size", "Current size", + [this] { return static_cast(myThing_.size()); }); +``` + +Counters use a `_total` suffix by convention. A histogram whose values can +exceed ~10,000 units (e.g. a microsecond duration beyond 10ms) still needs one +line added to `addMicrosecondHistogramView()` in `MetricsRegistry.cpp` -- the +only case that still touches a central file. There is no way to read a metric's +current value back from application code -- OTel's API is write-only by design; +keep your own state if your logic needs to both record and read a running value +(see the Doxygen header in `MetricMacros.h` and "Use Case 4" in +`tasks/metric-macro-plan.md` for the full explanation and the `prometheus-cpp` +contrast rationale). + +## Deployment Tiers + +Multiple xrpld instances can send telemetry to per-tier collectors that all +forward to one Grafana stack. Four resource attributes segregate the data so +one dashboard set serves every deployment: + +| Dimension | Attribute | Set by | Example values | +| ----------- | ------------------------ | ---------- | ------------------------------------------------ | +| Node | `service.instance.id` | xrpld cfg | `alice-laptop`, `ci-runner-7` | +| Service | `service.name` | xrpld cfg | `xrpld`, `xrpld-validator` | +| Network | `xrpl.network.type` | xrpld node | `mainnet`, `testnet`, `devnet`, `perf` | +| Environment | `deployment.environment` | collector | `local`, `test`, `ci`, `prod` | +| Work Item | `xrpl.work.item` | perf-iac | `RIPD-7455` (empty outside perf runs) | +| Branch | `xrpl.branch` | perf-iac | `baseline::`, `test::` | +| Node Role | `xrpl.node.role` | perf-iac | `validator`, `peer` | + +Dashboards expose these as the template variables `$node`, `$service_name`, +`$xrpl_network_type`, `$deployment_environment`, `$xrpl_work_item`, +`$xrpl_branch`, and `$xrpl_node_role` (each variable name matches its +Prometheus label). Select them top-down — work item → branch → node role → +node for a perf comparison run, or environment → network → service → node for +general use. Selecting **All** matches every value, including series lacking +the label, so mixed old/new data never disappears. + +The last three (`$xrpl_work_item`, `$xrpl_branch`, `$xrpl_node_role`) are +populated only during perf-iac comparison runs, which stamp them as resource +attributes from their own alloy pipeline. Outside those runs the labels are +absent; leaving the filters on **All** keeps every dashboard rendering +normally. + +### Who owns which attribute + +- **Node and service** come from xrpld config (`service_instance_id`, + `service_name`). Unique per process. +- **Network** is a property of the chain the node joined; the node derives it + from `[network_id]` and stamps `xrpl.network.type` on all three signals. +- **Environment** is a property of where the collector runs; each collector + serves one environment and stamps it. + +### The upsert vs insert rule + +The collector's `resource/tier` processor uses two actions on purpose: + +- `deployment.environment` → **`upsert`** (overwrite). The collector _is_ the + environment, so it is authoritative. +- `xrpl.network.type` → **`insert`** (fill only if absent). The node knows + its real network, so the collector must not overwrite it — `insert` only + supplies a value when the source did not (e.g. an older xrpld build). This + is what lets a local node connected to mainnet report `network=mainnet`, + not the collector's default. + +### Configuring a collector for a tier + +Each tier runs its own collector. Set the two values in the `resource/tier` +processor of the collector config (`otel-collector-config.yaml` for local +backends, `otel-collector-config.grafanacloud.yaml` for Grafana Cloud): + +```yaml +processors: + resource/tier: + attributes: + - key: deployment.environment + value: # local | test | ci | prod + action: upsert + - key: xrpl.network.type + value: # mainnet | testnet | devnet (fallback only) + action: insert +``` + +Suggested per-tier values: + +| Collector | `deployment.environment` | `xrpl.network.type` (fallback) | +| ------------------- | ------------------------ | ------------------------------ | +| Developer laptop | `local` | `devnet` | +| Test machines | `test` | `testnet` | +| CI runs | `ci` | `testnet` | +| Production observer | `prod` | `mainnet` | + +The `xrpl.network.type` value is only a fallback: when the node stamps its +own network (all current builds do), the node's value wins. Set it to the +network the collector most commonly serves. + +### How the tier labels reach metrics + +Resource attributes do not become Prometheus labels automatically. Two +collector settings make it work, both already enabled: + +- `prometheus.resource_to_telemetry_conversion: enabled: true` promotes + resource attributes to metric labels on the local scrape surface. +- `spanmetrics.resource_metrics_key_attributes` lists the tier attributes so + span-derived series stay grouped per node and tier. + +Traces and logs carry resource attributes natively; Grafana Cloud ingests all +three signals' attributes over OTLP directly. + +## Grafana Dashboards + +Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`: + +### RPC Performance (`rpc-performance`) + +| Panel | Type | PromQL | Labels Used | +| --------------------------- | ---------- | -------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| RPC Request Rate by Command | timeseries | `sum by (command) (rate(span_calls_total{span_name=~"rpc.command.*"}[5m]))` | `command` | +| RPC Latency p95 by Command | timeseries | `histogram_quantile(0.95, sum by (le, command) (rate(span_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m])))` | `command` | +| RPC Error Rate | bargauge | Error spans / total spans × 100, grouped by `command` | `command`, `status_code` | +| RPC Latency Heatmap | heatmap | `sum(increase(span_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m])) by (le)` | `le` (bucket boundaries) | +| Overall RPC Throughput | timeseries | `rpc.request` + `rpc.process` rate | — | +| RPC Success vs Error | timeseries | by `status_code` (UNSET vs ERROR) | `status_code` | +| Top Commands by Volume | bargauge | `topk(10, ...)` by `command` | `command` | +| WebSocket Message Rate | stat | `rpc.ws_message` rate | — | + +### Transaction Overview (`transaction-overview`) + +| Panel | Type | PromQL | Labels Used | +| ---------------------------------- | -------------- | -------------------------------------------------------------------------------------------- | ----------------------------------- | +| Transaction Processing Rate | timeseries | `rate(span_calls_total{span_name="tx.process"}[5m])` and `tx.receive` | `span_name` | +| Transaction Processing Latency | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="tx.process"})` | — | +| Transaction Path Distribution | piechart | `sum by (local) (rate(span_calls_total{span_name="tx.process"}[5m]))` | `local` | +| Transaction Receive vs Suppressed | timeseries | `rate(span_calls_total{span_name="tx.receive"}[5m])` | — | +| TX Processing Duration Heatmap | heatmap | `tx.process` histogram buckets | `le` | +| TX Apply Duration per Ledger | timeseries | p95/p50 of `tx.apply` | — | +| Peer TX Receive Rate | timeseries | `tx.receive` rate | — | +| TX Apply Failed Rate | stat | `rate(span_calls_total{span_name="tx.transactor",stage="apply",ter_result!~"tesSUCCESS\|"})` | `stage`, `ter_result` | +| TxQ Accept: Applied Ratio per Node | state-timeline | applied / (applied+failed) of `span_calls_total{span_name="txq.accept_tx"}` per node | `txq_status`, `service_instance_id` | + +### Consensus Health (`consensus-health`) + +| Panel | Type | PromQL | Labels Used | +| ----------------------------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| Consensus Round Duration | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="consensus.accept"})` | — | +| Consensus Proposals Sent Rate | timeseries | `rate(span_calls_total{span_name="consensus.proposal.send"}[5m])` | — | +| Ledger Close Duration | timeseries | `histogram_quantile(0.95, ... {span_name="consensus.round"})` (full round, not `consensus.ledger_close` which is only the sub-ms onClose prologue) | `consensus_mode` | +| Validation Send Rate | stat | `rate(span_calls_total{span_name="consensus.validation.send"}[5m])` | — | +| Ledger Apply Duration | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="consensus.accept.apply"})` | — | +| Close Time Agreement | timeseries | `rate(span_calls_total{span_name="consensus.accept.apply"}[5m])` | — | +| Consensus Mode Over Time | timeseries | `consensus.ledger_close` by `consensus_mode` | `consensus_mode` | +| Accept vs Close Rate | timeseries | `consensus.accept` vs `consensus.ledger_close` rate | — | +| Validation vs Close Rate | timeseries | `consensus.validation.send` vs `consensus.ledger_close` | — | +| Accept Duration Heatmap | heatmap | `consensus.accept` histogram buckets | `le` | + +### Ledger Operations (`ledger-operations`) + +| Panel | Type | PromQL | Labels Used | +| --------------------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------- | ----------- | +| Ledger Build Rate | stat | `ledger.build` call rate | — | +| Ledger Build Duration | timeseries | p95/p50 of `ledger.build` | — | +| Ledger Validation Rate | stat | `ledger.validate` call rate | — | +| Build Duration Heatmap | heatmap | `ledger.build` histogram buckets | `le` | +| TX Apply Duration | timeseries | p95/p50 of `tx.apply` | — | +| TX Apply Rate | timeseries | `tx.apply` call rate | — | +| Ledger Store Rate | stat | `ledger.store` call rate | — | +| Build vs Close Duration | timeseries | p95 `ledger.build` vs `consensus.round` (full round, not `consensus.ledger_close` which is only the sub-ms onClose prologue) | — | +| Ledger Close Interval & Age | timeseries | Interval: `1/rate(ledgers_closed_total)`; Age: `time() - (server_info{metric="last_close_time"} + 946684800)` | — | + +### Peer Network (`peer-network`) + +Requires `trace_peer=1` in the `[telemetry]` config section. + +| Panel | Type | PromQL | Labels Used | +| -------------------------------- | ---------- | ------------------------------ | -------------------- | +| Proposal Receive Rate | timeseries | `peer.proposal.receive` rate | — | +| Validation Receive Rate | timeseries | `peer.validation.receive` rate | — | +| Proposals Trusted vs Untrusted | piechart | by `proposal_trusted` | `proposal_trusted` | +| Validations Trusted vs Untrusted | piechart | by `validation_trusted` | `validation_trusted` | + +### Node Health -- System Metrics (`node-health`) + +| Panel | Type | PromQL | Labels Used | +| -------------------------------------- | ---------- | ---------------------------------------------------------- | ---------------- | +| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | +| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | +| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | +| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | +| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | +| Job Queue Depth | timeseries | `jobq_job_count` | — | +| Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | +| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | +| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | +| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | +| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | +| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | +| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | +| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | +| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"", quantile="$quantile"}` | `quantile` | +| All Jobs Dequeue Wait (Detail) | timeseries | `{__name__=~"_q", quantile="$quantile"}` | `quantile` | +| Server State | stat | `server_info{metric="server_state"}` | `metric` | +| Uptime | stat | `server_info{metric="uptime"}` | `metric` | +| Peer Count | stat | `server_info{metric="peers"}` | `metric` | +| Validated Ledger Seq | stat | `server_info{metric="validated_ledger_seq"}` | `metric` | +| Build Version | stat | `build_info` | `version` | +| Complete Ledger Ranges | table | `complete_ledgers` | `bound`, `index` | +| Database Sizes | timeseries | `db_metrics{metric=~"db_kb_.*"}` | `metric` | +| Historical Fetch Rate | stat | `db_metrics{metric="historical_perminute"}` | `metric` | + +### Network Traffic -- System Metrics (`network-traffic`) + +| Panel | Type | PromQL | Labels Used | +| ------------------------------------ | ---------- | -------------------------------------------------------------------------------------------------------------------------- | ----------- | +| Active Peers | timeseries | `peer_finder_active_*_peers` | — | +| Peer Disconnects | timeseries | `increase(overlay_peer_disconnects[$__rate_interval])` | — | +| Total Network Bytes | timeseries | `rate(total_bytes_in/out[$__rate_interval])` | — | +| Total Network Messages | timeseries | `rate(total_messages_in/out[$__rate_interval])` | — | +| Transaction Traffic | timeseries | `rate(transactions_messages_in/out[$__rate_interval])` | — | +| Proposal Traffic | timeseries | `rate(proposals_messages_in/out[$__rate_interval])` | — | +| Validation Traffic | timeseries | `rate(validations_messages_in/out[$__rate_interval])` | — | +| Traffic by Category | bargauge | `topk(10, label_replace(sum by (service_instance_id)(rate([$__rate_interval])),"__name__","","","") or …)` | — | +| Duplicate Traffic (Wasted Bandwidth) | timeseries | `rate(*_duplicate_bytes_in/out[$__rate_interval])` | — | +| All Traffic Categories (Detail) | timeseries | `topk(15, label_replace(sum by (service_instance_id)(rate([$__rate_interval])),"__name__","","","") or …)` | — | + +> **Why the per-category panels enumerate each metric.** A bare +> `rate({__name__=~".*_bytes_in"}[…])` fails on Mimir/Cloud with _"vector +> cannot contain metrics with the same labelset"_: `rate()` drops the +> `__name__` label, so the many matched counters collapse to identical +> labelsets. Wrapping in `sum by (__name__, …)` does **not** help (the inner +> vector is rejected before the outer `sum`). The working form enumerates each +> `*_bytes_in` metric and re-attaches its name with `label_replace(..., +"__name__", "", "", "")`, so the existing `{{__name__}}` legend and +> the per-series display-name overrides keep working. + +### RPC & Pathfinding -- System Metrics (`rpc-pathfinding`) + +| Panel | Type | PromQL | Labels Used | +| ------------------------- | ---------- | ------------------------------------------------ | ----------- | +| RPC Request Rate | stat | `rate(rpc_requests[5m])` | — | +| RPC Response Time | timeseries | `histogram_quantile(0.95, rpc_time_bucket)` | — | +| RPC Response Size | timeseries | `histogram_quantile(0.95, rpc_size_bucket)` | — | +| RPC Response Time Heatmap | heatmap | `rpc_time_bucket` | — | +| Pathfinding Fast Duration | timeseries | `histogram_quantile(0.95, pathfind_fast_bucket)` | — | +| Pathfinding Full Duration | timeseries | `histogram_quantile(0.95, pathfind_full_bucket)` | — | +| Resource Warnings Rate | stat | `rate(warn[5m])` | — | +| Resource Drops Rate | stat | `rate(drop[5m])` | — | + +### Span → Metric → Dashboard Summary + +| Span Name | Prometheus Metric Filter | Grafana Dashboard | +| ------------------------------ | -------------------------------------------- | --------------------------------------------- | +| `rpc.http_request` | `{span_name="rpc.http_request"}` | RPC Performance (Overall Throughput) | +| `rpc.ws_upgrade` | `{span_name="rpc.ws_upgrade"}` | -- (available but not paneled) | +| `rpc.ws_message` | `{span_name="rpc.ws_message"}` | RPC Performance (WebSocket Rate) | +| `rpc.process` | `{span_name="rpc.process"}` | RPC Performance (Overall Throughput) | +| `rpc.command.*` | `{span_name=~"rpc.command.*"}` | RPC Performance (Rate, Latency, Error, Top) | +| `tx.process` | `{span_name="tx.process"}` | Transaction Overview (Rate, Latency, Heatmap) | +| `tx.receive` | `{span_name="tx.receive"}` | Transaction Overview (Rate, Receive) | +| `tx.apply` | `{span_name="tx.apply"}` | Transaction Overview + Ledger Ops (Apply) | +| `txq.enqueue` | `{span_name="txq.enqueue"}` | -- (available but not paneled) | +| `txq.apply_direct` | `{span_name="txq.apply_direct"}` | -- (available but not paneled) | +| `txq.batch_clear` | `{span_name="txq.batch_clear"}` | -- (available but not paneled) | +| `txq.accept` | `{span_name="txq.accept"}` | -- (available but not paneled) | +| `txq.accept_tx` | `{span_name="txq.accept_tx"}` | -- (available but not paneled) | +| `txq.cleanup` | `{span_name="txq.cleanup"}` | -- (available but not paneled) | +| `consensus.round` | `{span_name="consensus.round"}` | -- (available but not paneled) | +| `consensus.phase.open` | `{span_name="consensus.phase.open"}` | -- (available but not paneled) | +| `consensus.establish` | `{span_name="consensus.establish"}` | -- (available but not paneled) | +| `consensus.update_positions` | `{span_name="consensus.update_positions"}` | -- (available but not paneled) | +| `consensus.check` | `{span_name="consensus.check"}` | -- (available but not paneled) | +| `consensus.accept` | `{span_name="consensus.accept"}` | Consensus Health (Duration, Rate, Heatmap) | +| `consensus.proposal.send` | `{span_name="consensus.proposal.send"}` | Consensus Health (Proposals Rate) | +| `consensus.ledger_close` | `{span_name="consensus.ledger_close"}` | Consensus Health (Close, Mode) | +| `consensus.validation.send` | `{span_name="consensus.validation.send"}` | Consensus Health (Validation Rate) | +| `consensus.accept.apply` | `{span_name="consensus.accept.apply"}` | Consensus Health (Apply Duration, Close Time) | +| `consensus.mode_change` | `{span_name="consensus.mode_change"}` | -- (available but not paneled) | +| `consensus.proposal.receive` | `{span_name="consensus.proposal.receive"}` | -- (available but not paneled) | +| `consensus.validation.receive` | `{span_name="consensus.validation.receive"}` | -- (available but not paneled) | +| `ledger.build` | `{span_name="ledger.build"}` | Ledger Ops (Build Rate, Duration, Heatmap) | +| `ledger.validate` | `{span_name="ledger.validate"}` | Ledger Ops (Validation Rate) | +| `ledger.store` | `{span_name="ledger.store"}` | Ledger Ops (Store Rate) | +| `peer.proposal.receive` | `{span_name="peer.proposal.receive"}` | Peer Network (Rate, Trusted/Untrusted) | +| `peer.validation.receive` | `{span_name="peer.validation.receive"}` | Peer Network (Rate, Trusted/Untrusted) | + +## Alerting + +xrpld provisions six Grafana alert rules on the health-critical metrics, so a +stock stack alerts out of the box with no UI setup. Rules are provisioned from +`docker/telemetry/grafana/provisioning/alerting/` and load automatically when +the Grafana container starts. They appear under **Alerting → Alert rules**, +folder **xrpld**. + +### Alert catalogue + +All rules evaluate every minute against the Prometheus datasource, over a +5-minute window, and group by `service_instance_id` so each node alerts on its +own. Alerts fire only after the condition holds for the `for` dwell time. + +| Alert | Severity | Fires when | For | +| ----------------------- | -------- | ----------------------------------------- | --- | +| `LedgerHistoryMismatch` | critical | `rate(ledger_history_mismatch_total)` > 0 | 5m | +| `LedgerCloseStalled` | critical | `rate(ledgers_closed_total)` ≈ 0 | 3m | +| `ValidationsMissed` | warning | `rate(validation_missed_total)` > 0 | 5m | +| `ValidationsNotChecked` | warning | `rate(validations_checked_total)` ≈ 0 | 5m | +| `JobQueueTxOverflow` | warning | `rate(jq_trans_overflow_total)` > 0 | 5m | +| `JobQueueLatencyHigh` | warning | p99 `job_queued_us` > 1s | 5m | + +#### Consensus / ledger health + +**LedgerHistoryMismatch** — The node closed a ledger whose history diverges +from the validated network chain. Likely causes: corrupted local state, a bug, +or a node that fell out of sync and rebuilt incorrectly. Investigate the node's +ledger acquisition logs; a healthy node never mismatches. + +**LedgerCloseStalled** — No ledgers closed for 3 minutes. A healthy node closes +one every ~3-5s. Likely causes: lost peer connectivity, consensus stall, or the +process is hung. This rule also fires on _NoData_ — if the series disappears the +node is likely down. Check peer count and process health first. + +#### Validator health + +**ValidationsMissed** — This validator's validations are not agreeing with the +validated ledger. Sustained misses risk removal from UNLs. Check clock sync, +peer connectivity, and whether the node is keeping up with ledger close. + +**ValidationsNotChecked** — The node has stopped checking incoming validations +from peers. Likely causes: overlay/peer disconnection or a stalled validation +pipeline. Fires on NoData as well. + +#### Job queue / resource health + +**JobQueueTxOverflow** — The transaction job queue is full and transactions are +being dropped. The node is shedding load it cannot process. Check CPU, the +`JobQueueLatencyHigh` alert, and offered load. + +**JobQueueLatencyHigh** — p99 queue wait exceeds 1 second, i.e. jobs back up +before running. The node is saturated. Correlate with CPU and the Job Queue +dashboard. + +### Tuning thresholds + +Thresholds live in +`docker/telemetry/grafana/provisioning/alerting/rules.yaml` as the `params` +array of each rule's `C` (threshold) node. Common tunables: + +- **`JobQueueLatencyHigh`** — `params: [1000000]` is 1 000 000 µs (1s). Lower + it for latency-sensitive deployments. +- **`LedgerCloseStalled` / `ValidationsNotChecked`** — use `lt` with a tiny + epsilon (`0.001`) rather than `0`, so floating-point rate noise near zero + does not suppress the alert. + +Edit the file and restart the Grafana container to reload: + +```bash +docker compose -f docker/telemetry/docker-compose.yml restart grafana +``` + +### Sending alerts somewhere real + +Two contact points are provisioned in +`docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml`: + +| Contact point | Receivers | Gets | +| ---------------- | ------------- | ------------------------ | +| `xrpld-default` | Slack | warning-severity alerts | +| `xrpld-critical` | Slack + email | critical-severity alerts | + +The severity split lives in +`docker/telemetry/grafana/provisioning/alerting/policies.yaml`: the root route +sends everything to `xrpld-default`, and a child route matching +`severity = critical` overrides to `xrpld-critical`. So a critical alert goes +to Slack **and** email; a warning goes to Slack only. Both group by +`alertname` + `service_instance_id`; critical alerts re-page hourly vs the 4h default. + +#### Configure delivery (no secrets in git) + +The Slack webhook and email address are **not** hard-coded — the YAML +references `${SLACK_WEBHOOK_URL}` and `${ALERT_EMAIL_TO}`, which Grafana +expands from the environment at startup. Supply them through a gitignored +env file: + +```bash +cp docker/telemetry/.env.alerting.example docker/telemetry/.env.alerting +# edit .env.alerting — this file is gitignored, never commit the webhook/address +docker compose -f docker/telemetry/docker-compose.yml up -d grafana +``` + +- **Slack** — set `SLACK_WEBHOOK_URL` to an incoming-webhook URL. Drives both + tiers. +- **Email** — set `ALERT_EMAIL_TO` (comma-separated) **and** point the + `GF_SMTP_*` vars at a real relay with `GF_SMTP_ENABLED=true`. Grafana can + only send mail once SMTP is configured. + +Any variable left blank disables that path; the stack still runs. To add a +third destination (PagerDuty, Opsgenie, a custom webhook), add a receiver to +the relevant contact point. + +### Verifying alert provisioning loaded + +After the stack is up: + +```bash +# All six rules present? +curl -s http://localhost:3000/api/v1/provisioning/alert-rules | jq '.[].title' + +# Contact points present? +curl -s http://localhost:3000/api/v1/provisioning/contact-points | jq '.[].name' +``` + +Grafana logs a provisioning error and skips the file if the YAML is malformed: + +```bash +docker compose -f docker/telemetry/docker-compose.yml logs grafana | grep -i alerting +``` + +## Log-Trace Correlation + +When xrpld is built with `telemetry=ON`, log lines emitted within an active OpenTelemetry span automatically include `trace_id` and `span_id` fields: + +``` +2024-Jan-15 10:30:45.123456 UTC LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42 +``` + +This enables bidirectional navigation between logs and traces in Grafana: + +- **Tempo -> Loki**: Click "Logs for this trace" on any trace in Grafana Tempo to see all log lines from that trace. +- **Loki -> Tempo**: Click the `TraceID` derived field link on any log line containing `trace_id=` to jump to the full trace in Tempo. + +### Log Ingestion Pipeline + +Log files are ingested by the OTel Collector's `filelog` receiver, which tails `debug.log` files and parses them with a regex that extracts `timestamp`, `partition`, `severity`, `trace_id`, `span_id`, and `message` fields. Parsed entries are exported to Grafana Loki. + +The receiver tails `/var/log/xrpld/*/debug.log` inside the collector container. docker-compose bind-mounts the host log root there; the source defaults to the repo-relative `docker/telemetry/data/logs`, which the telemetry configs write to (`data/logs//debug.log`) and which needs no root. To tail logs from elsewhere, set `XRPLD_LOG_DIR` before `docker compose up` (the integration test does this to point at its own workdir). The single trailing `*` matches one per-network or per-node subdirectory. + +### LogQL Query Examples + +The OTel Collector emits logs to Loki with `service_name="xrpld"` (not `job="xrpld"`). + +```logql +# Find all logs for a specific trace +{service_name="xrpld"} |= "trace_id=abc123def456789012345678abcdef01" + +# Error logs with trace context (log lines with ERR severity that have a trace_id) +{service_name="xrpld"} |= "ERR" |= "trace_id=" + +# All logs from a specific partition that were emitted during a span +{service_name="xrpld"} |= "LedgerMaster" | regexp `trace_id=(?P[a-f0-9]+)` | trace_id != "" + +# Logs from a specific subsystem during a span (e.g. LedgerConsensus) +{service_name="xrpld"} |= "LedgerConsensus" |= "trace_id=" + +# Logs from the last hour containing trace context +{service_name="xrpld"} |= "trace_id=" | regexp `(?P\S+):(?P\S+)\s+trace_id=(?P[a-f0-9]+)` + +# Count of traced vs untraced log lines +count_over_time({service_name="xrpld"} |= "trace_id=" [5m]) +``` + +### Verifying Log Correlation + +1. Start the observability stack and xrpld with telemetry enabled. +2. Send an RPC request: `curl http://localhost:5005 -d '{"method":"server_info"}'` +3. Check the debug.log for `trace_id=` entries: `grep trace_id= /path/to/debug.log` +4. Open Grafana at http://localhost:3000 -> Explore -> Loki and search for `{service_name="xrpld"} |= "trace_id="`. +5. Click the TraceID link to navigate to the corresponding trace in Tempo. + +## Troubleshooting + +### No traces appearing in Tempo + +1. Check xrpld logs for `Telemetry starting` message +2. Verify `enabled=1` in the `[telemetry]` config section +3. Test collector connectivity: `curl -v http://localhost:4318/v1/traces` +4. Check collector logs: `docker compose -f docker/telemetry/docker-compose.yml logs otel-collector` +5. Verify Tempo is receiving data: open Grafana → Explore → select Tempo datasource → search by `service.name = xrpld` +6. Check Tempo logs: `docker compose -f docker/telemetry/docker-compose.yml logs tempo` + +### No system metrics in Prometheus + +1. Check xrpld logs for `OTelCollector starting` message +2. Verify `server=otel` in the `[insight]` config section +3. Verify the endpoint in `[insight]` points to the OTLP/HTTP port (default: `http://localhost:4318/v1/metrics`) +4. Check that the `otlp` receiver is in the metrics pipeline receivers in `otel-collector-config.yaml` +5. Query Prometheus directly: `curl 'http://localhost:9090/api/v1/query?query=jobq_job_count'` + +### Server info gauge shows server_state=0 + +This is normal during startup. The server starts in DISCONNECTED mode (0) and +progresses through CONNECTED (1), SYNCING (2), TRACKING (3), to FULL (4). +Wait for the node to sync with the network. + +### Database metrics showing zero + +The `getKBUsed*()` methods require SQLite databases to exist. If running with +`--standalone` or before the first ledger is stored, these will be zero. + +### Slow TMGetObjectByHash service + +Use this when a peer reports slow object fetches, or when `job_queued_us` / +`job_running_us` for `job_type="ledgerRequest"` rises. The goal is to name the +cause, not to confirm the slowness. + +End-to-end handler time splits into three additive parts, and each has its own +signal: + +```mermaid +flowchart LR + A["`**Request arrives** + onMessage + TMGetObjectByHash`"] --> B["`**1. Queue wait** + job_queued_us + handler=RcvGetObjByHash`"] + B --> C["`**2. NodeStore lookup** + getobject_lookup_us`"] + C --> D["`**3. Everything else** + protobuf, serialization, + reply construction`"] + D --> E["`**Reply sent**`"] + + C -.-> F["`job_running_us + handler=RcvGetObjByHash + covers steps 2 + 3`"] + D -.-> F + + style A fill:#1f4e79,color:#ffffff + style B fill:#7b3f00,color:#ffffff + style C fill:#2d5016,color:#ffffff + style D fill:#4a148c,color:#ffffff + style E fill:#1f4e79,color:#ffffff + style F fill:#37474f,color:#ffffff +``` + +**Reading the diagram** + +- Step 1 is time the job sat in `JtLedgerReq` before a worker picked it up. Only + `job_queued_us` measures it. +- Steps 2 and 3 both happen inside the worker, so `job_running_us` covers them + together. `getobject_lookup_us` isolates step 2 alone. +- Step 3 is therefore not measured directly. Derive it: + `job_running_us − getobject_lookup_us`. That subtraction is what makes the set + able to name a cause instead of just reporting a duration. + +**Procedure** — work through these in order. The first row that matches is the +answer. + +> **Scope every query to one node.** All snippets below carry +> `service_instance_id="$node"`. On a shared Grafana stack an unscoped selector +> aggregates across every node and branch reporting to it, so another node's +> saturation would be attributed to this one. Substitute the node's public key +> for `$node` when querying Prometheus directly rather than from a dashboard. + +1. Split queue wait from run time. Compare the two p99s for the handler: + + ```promql + histogram_quantile(0.99, sum by (le) (rate(job_queued_us_bucket{handler="RcvGetObjByHash", service_instance_id="$node"}[5m]))) + histogram_quantile(0.99, sum by (le) (rate(job_running_us_bucket{handler="RcvGetObjByHash", service_instance_id="$node"}[5m]))) + ``` + +2. If run time is the larger term, split it against the fetch loop: + + ```promql + histogram_quantile(0.99, sum by (le) (rate(getobject_lookup_us_bucket{service_instance_id="$node"}[5m]))) + ``` + +3. Match the outcome below. + +| Observation | Root cause and next step | +| -------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `job_queued_us{handler="RcvGetObjByHash"}` high, `job_running_us` normal | **Queue contention** — the work is cheap, the wait is not. Confirm with `jobq_ledgerrequest_deferred{service_instance_id="$node"} > 0`, then compare `job_queued_us{handler="RcvGetLedger"}`: if it is also high, both producers are starved by the limit of 3, not by each other. | +| `job_running_us` high and within ~10% of `getobject_lookup_us` | **NodeStore is the bottleneck** — nearly all run time is in the fetch loop. Check `rate(getobject_lookups_total{result="miss"}[5m])` and the existing NuDB / `nodestore_state` panels. A miss-heavy mix means real disk seeks. | +| `job_running_us` high but `getobject_lookup_us` low | **Cost is outside the fetch loop** — protobuf, serialization, or reply construction. Storage is fine. Look at reply size: a large `getobject_request_objects` with a high hit rate means big replies to build and send. | +| `getobject_request_objects` p99 large | **Peers are sending big batches** — the work is real, not a regression. Nothing is broken; the node is being asked to do more. Decide whether to accept the load or price it higher. | +| `rate(getobject_rejected_total{reason="oversize"}[5m])` rising | **Non-conforming traffic** — requests above `kHardMaxReplyNodes` are being refused before any NodeStore access. Check `getobject_charge` to confirm the pricing escalates for the requests that _are_ accepted. | +| `rate(getobject_rejected_total{reason="malformed_ledgerhash"}[5m])` rising | **Malformed requests** — a peer is sending a ledgerhash that is not 32 bytes. Refused at the gate; no queue or storage cost incurred. | +| All GetObject metrics normal, `jobq_*_deferred` high on another type | **This path is exonerated** — the slowness is elsewhere. Find the saturated type with `topk(5, {__name__=~"jobq_.*_deferred", service_instance_id="$node"} > 0)` and investigate that producer instead. | + +Row 2 says "within ~10%", not "equal", deliberately: `job_running_us` also +covers the charge computation (PeerImp.cpp:2757) and the reply `send()` that +follow the loop, so it is always the larger of the two. Treat a small residual as +normal and only a large one as a signal — that is what row 3 is for. + +The last row matters as much as the others: the set can rule this path _out_, +which a slowness-only metric cannot. + +**Caveats** + +- `handler` collapses to `"other"` for any job name that is not all ASCII + letters, so a handler breakdown is not exhaustive — see + [The `handler` Label](#the-handler-label). This does not affect + `RcvGetObjByHash` or `RcvGetLedger`; both pass through as distinct series. +- `jobq_*_deferred` is sampled at 1 s. A zero reading does not prove no + saturation occurred — see the sampling caveat under + [Per-Job-Type Queue Saturation](#per-job-type-queue-saturation). +- The two families compared in this procedure are exported on **different + cadences by separate meter providers**: the `jobq_*` gauges every 1 s + (`Telemetry.cpp` global provider) and `job_queued_us` / `job_running_us` / + `getobject_*` every 10 s (the private `MetricsRegistry` provider). A short + spike can therefore appear in one family a step before the other. When + correlating them, widen the window rather than reading a single scrape, and + do not conclude the two disagree from one interval's difference. +- A request rejected at either gate contributes to no other GetObject metric, so + a rejection spike will _not_ show up as latency. Check the rejection counters + before concluding that traffic is normal. + +### High memory usage + +- Reduce trace volume with collector-side tail sampling (xrpld head sampling is + fixed at 1.0 and is not configurable) +- Reduce `max_queue_size` and `batch_size` +- Disable high-volume trace categories: `trace_peer=0` + +### Collector connection failures + +- Verify endpoint URL matches collector address +- Check firewall rules for ports 4317/4318 +- If using TLS, verify certificate path with `tls_ca_cert` + +### No trace_id in log output + +- Verify xrpld was built with `telemetry=ON` (the `XRPL_ENABLE_TELEMETRY` preprocessor flag) +- Verify `enabled=1` in the `[telemetry]` config section +- Log lines only contain `trace_id`/`span_id` when emitted inside an active span — background logs outside of RPC/consensus/transaction processing will not have trace context +- Check that the specific trace category is enabled (e.g., `trace_rpc=1`) + +### No logs in Loki + +- Verify the log file mount in docker-compose.yml points to the correct xrpld log directory (default source `docker/telemetry/data/logs`, or the `XRPLD_LOG_DIR` override) and that xrpld actually writes `debug.log` there +- Check OTel Collector logs for filelog receiver errors: `docker compose logs otel-collector` +- Verify Loki is running: `curl http://localhost:3100/ready` +- Check the filelog receiver glob `/var/log/xrpld/*/debug.log` matches your log layout — the log file must sit one subdirectory below the mount root + +## Performance Tuning + +| Scenario | Recommendation | +| ------------------------ | --------------------------------------------------------- | +| Production mainnet | `trace_peer=0`; reduce volume via collector tail sampling | +| Testnet/devnet | Full tracing (head sampling fixed at 1.0) | +| Debugging specific issue | Full tracing (head sampling fixed at 1.0) | +| High-throughput node | Increase `batch_size=1024`, `max_queue_size=4096` | + +## Disabling Telemetry + +Set `enabled=0` in config (runtime disable) or build without the flag: + +```bash +cmake --preset default -Dtelemetry=OFF +``` + +When telemetry is compiled out, all trace macros expand to no-ops with zero overhead. + +## Validating Telemetry Stack + +After deploying telemetry, use the Phase 10 workload tools to validate the full stack end-to-end. + +### Quick Validation + +```bash +# Run the full validation suite (starts cluster, generates load, validates): +docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld + +# Check the report: +cat /tmp/xrpld-validation/reports/validation-report.json | jq '.summary' +``` + +### What Gets Validated + +| Category | Checks | Description | +| ---------- | -------------- | ------------------------------------------------------- | +| Spans | 16+ span types | All span names appear in Tempo with required attributes | +| Metrics | 30+ metrics | SpanMetrics, StatsD gauges/counters, Phase 9 metrics | +| Logs | 2 checks | trace_id/span_id present in Loki, cross-reference works | +| Dashboards | 10 dashboards | All Grafana dashboards load without errors | + +### Running Individual Tools + +```bash +# RPC load only: +python3 docker/telemetry/workload/rpc_load_generator.py \ + --endpoints ws://localhost:6006 --rate 50 --duration 120 + +# Transaction mix only: +python3 docker/telemetry/workload/tx_submitter.py \ + --endpoint ws://localhost:6006 --tps 5 --duration 120 + +# Validation only (assumes load already ran): +python3 docker/telemetry/workload/validate_telemetry.py \ + --report /tmp/report.json +``` + +### Interpreting Failures + +- **Span failures**: Check that the relevant trace category is enabled in `[telemetry]` config (e.g., `trace_rpc=1`). +- **Metric failures**: Verify the OTel Collector is running and Prometheus is scraping port 8889. Check `docker compose logs otel-collector`. +- **Dashboard failures**: Ensure Grafana provisioning is mounted correctly. Check `docker compose logs grafana`. + +## Performance Benchmarking + +Measure the overhead of the telemetry stack against a baseline: + +```bash +docker/telemetry/workload/benchmark.sh --xrpld .build/xrpld --duration 300 +``` + +### Benchmark Thresholds + +| Metric | Target | Description | +| ----------------- | ------ | -------------------------------------- | +| CPU overhead | < 3% | Average CPU increase across nodes | +| Memory overhead | < 5MB | Peak RSS increase per node | +| RPC p99 latency | < 2ms | Additional p99 latency for server_info | +| Throughput impact | < 5% | Reduction in ledger close rate | +| Consensus impact | < 1% | Increase in consensus round time | + +### Tuning for Production + +If benchmarks exceed thresholds: + +1. **Reduce sampling**: `sampling_ratio=0.01` (1% of traces) +2. **Disable peer tracing**: `trace_peer=0` (highest volume category) +3. **Increase batch delay**: `batch_delay_ms=10000` (less frequent exports) +4. **Reduce queue size**: `max_queue_size=1024` (back-pressure earlier) + +See `docker/telemetry/workload/README.md` for full documentation. diff --git a/include/xrpl/beast/insight/OTelCollector.h b/include/xrpl/beast/insight/OTelCollector.h new file mode 100644 index 00000000000..e043531f997 --- /dev/null +++ b/include/xrpl/beast/insight/OTelCollector.h @@ -0,0 +1,135 @@ +#pragma once + +/** + * @file OTelCollector.h + * @brief OpenTelemetry-based implementation of the beast::insight::Collector + * interface for native OTLP metric export. + * + * When XRPL_ENABLE_TELEMETRY is defined, OTelCollector maps each + * beast::insight instrument type (Counter, Gauge, Event, Meter, Hook) to + * the corresponding OpenTelemetry Metrics SDK instrument and exports + * them via OTLP/HTTP to an OpenTelemetry Collector. + * + * When XRPL_ENABLE_TELEMETRY is NOT defined, OTelCollector::New() returns + * a NullCollector so the binary compiles without OTel dependencies. + * + * Dependency diagram: + * + * +-----------------+ +-------------------+ + * | Collector (ABC) |<----| OTelCollector | + * +-----------------+ | (public header) | + * ^ +-------------------+ + * | | + * +-----------------+ +-------------------+ + * | NullCollector | | OTelCollectorImp | + * | (fallback when | | (impl in .cpp, | + * | no telemetry) | | uses OTel SDK) | + * +-----------------+ +-------------------+ + * | + * +-------------------+ + * | OTel Metrics SDK | + * | MeterProvider | + * | OTLP HTTP Metric | + * | Exporter | + * +-------------------+ + */ + +#include +#include + +#include +#include + +namespace beast::insight { + +/** + * @brief A Collector that exports metrics via OpenTelemetry OTLP/HTTP. + * + * Replaces StatsD-based metric collection with native OTel Metrics SDK + * instruments. Each beast::insight instrument maps to an OTel equivalent: + * + * - Counter -> OTel Counter + * - Gauge -> OTel ObservableGauge (async callback) + * - Event -> OTel Histogram (duration in milliseconds) + * - Meter -> OTel Counter (monotonic, unsigned) + * - Hook -> Called by PeriodicMetricReader at collection time + * + * Example — primary use (create the collector and record a metric): + * @code + * auto collector = beast::insight::OTelCollector::New( + * "http://localhost:4318/v1/metrics", // OTLP/HTTP endpoint + * "xrpld", // metric name prefix + * "node-1", // service.instance.id + * "xrpld", // service.name + * "mainnet", // xrpl.network.type + * journal); + * + * auto counter = collector->makeCounter("ledgers", "closed"); + * ++counter; // exported on the next PeriodicMetricReader tick + * @endcode + * + * Example — edge case (telemetry disabled at compile time): New() + * returns a NullCollector, so callers need no #ifdef guard. The same + * instruments compile and run, but recording is a no-op. + * @code + * auto collector = beast::insight::OTelCollector::New( + * endpoint, prefix, instanceId, serviceName, networkType, journal); + * auto gauge = collector->makeGauge("peers", "count"); + * gauge = 42; // silently discarded when XRPL_ENABLE_TELEMETRY is off + * @endcode + * + * @note Thread safety: instrument recording (Counter::Add, + * Histogram::Record, and atomic gauge writes) is thread-safe and + * may be called concurrently. Instrument *creation* (make_counter, + * make_gauge, etc.) is serialized by an internal mutex. Hook and + * observable-gauge callbacks run on the SDK's collection thread, so + * any state they read must itself be thread-safe. + * @note Limitations: metrics export over OTLP/HTTP only (no gRPC); the + * PeriodicMetricReader interval is fixed at 1s; and gauge values + * are stored as int64_t, so fractional gauges are truncated. + * + * @see StatsDCollector for the StatsD-based alternative. + * @see NullCollector for the no-op fallback. + */ +class OTelCollector : public Collector +{ +public: + explicit OTelCollector() = default; + + /** + * @brief Factory method to create an OTelCollector instance. + * + * When XRPL_ENABLE_TELEMETRY is defined, creates a real OTel-backed + * collector that exports metrics via OTLP/HTTP. When telemetry is + * disabled at compile time, returns a NullCollector. + * + * @param endpoint OTLP/HTTP metrics endpoint URL + * (e.g. "http://localhost:4318/v1/metrics"). + * @param prefix Prefix prepended to all metric names + * (e.g. "xrpld"). + * @param instanceId Unique identifier for this node instance, + * emitted as the `service.instance.id` OTel + * resource attribute. Defaults to empty string + * (attribute omitted when empty). + * @param serviceName Value for the `service.name` OTel resource + * attribute. When empty, defaults to "xrpld". + * Matches the trace exporter's service.name so + * metrics and traces share one service identity. + * @param networkType Value for the `xrpl.network.type` OTel resource + * attribute (e.g. "mainnet"). When empty, the + * attribute is omitted. Supplied by the node so + * metrics carry the same network label as traces. + * @param journal Journal for logging. + * @return Shared pointer to the created Collector. + */ + static std::shared_ptr + // NOLINTNEXTLINE(readability-identifier-naming) + New(std::string const& endpoint, + std::string const& prefix, + std::string const& instanceId, + std::string const& serviceName, + std::string const& networkType, + Journal journal); +}; + +} // namespace beast::insight diff --git a/include/xrpl/core/Job.h b/include/xrpl/core/Job.h index 93b39701bed..53e8566a99b 100644 --- a/include/xrpl/core/Job.h +++ b/include/xrpl/core/Job.h @@ -109,6 +109,20 @@ class Job : public CountedObject [[nodiscard]] JobType getType() const; + /** + * Returns the job name supplied to JobQueue::addJob. + * + * Several job types have more than one producer (for example both + * RcvGetLedger and RcvGetObjByHash run as JtLedgerReq), so the name is + * the only thing that tells them apart. It is read by the JobQueue + * PerfLog hooks and exported as the `handler` metric label. + * + * Empty for the default and index-only constructors, which carry no + * name; callers must treat an empty name as "unknown". + */ + [[nodiscard]] std::string const& + getName() const; + /** * Returns the time when the job was queued. */ diff --git a/include/xrpl/core/JobQueue.h b/include/xrpl/core/JobQueue.h index 0c9fc76357f..e07ab0362c9 100644 --- a/include/xrpl/core/JobQueue.h +++ b/include/xrpl/core/JobQueue.h @@ -334,7 +334,9 @@ class JobQueue : private Workers::Callback // any. // // Invariants: - // + // The calling thread owns the JobLock. This function mutates the + // deferred and running counts, which mutex_ guards; collect() reads + // them under the same lock. void finishJob(JobType type); diff --git a/include/xrpl/core/JobTypeData.h b/include/xrpl/core/JobTypeData.h index d53440e1cab..b503cbcbbdd 100644 --- a/include/xrpl/core/JobTypeData.h +++ b/include/xrpl/core/JobTypeData.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,23 @@ struct JobTypeData beast::insight::Collector::ptr collector_; public: + /** + * Metric-name suffixes appended to `JobTypeInfo::name()`. + * + * The constructor builds the instrument names from these. Public so + * tests can assert on the exported names without repeating the + * literals. The collector is the `"jobq"` group, so + * `GroupImp::makeName()` prefixes `jobq.` and `OTelCollector` + * lowercases and turns `.` into `_`: `ledgerRequest` + + * `kSuffixDeferred` exports as `jobq_ledgerrequest_deferred`. + */ + /** @{ */ + static constexpr char kSuffixWaiting[] = "_waiting"; + static constexpr char kSuffixRunning[] = "_running"; + static constexpr char kSuffixDeferred[] = "_deferred"; + static constexpr char kSuffixQueued[] = "_q"; + /** @} */ + /* The job category which we represent */ JobTypeInfo const& info; @@ -36,6 +54,32 @@ struct JobTypeData beast::insight::Event dequeue; beast::insight::Event execute; + /** + * Saturation gauges, published by `JobQueue::collect()`. + * + * Each mirrors the same-named counter above so per-job-type queue + * pressure is visible in metrics. Without them the only exported queue + * signal is the process-wide `jobq_job_count`, which cannot attribute + * pressure to a job type. + * + * `waitingGauge` is the backlog not yet started, `runningGauge` is the + * in-flight count, and `deferredGauge` is the count held back by this + * type's concurrency limit. `deferredGauge` is the leading indicator: + * `JobQueue::addJob()` never rejects, so a capped type under pressure + * shows up as latency only after the fact, whereas a non-zero deferred + * reading precedes it. + * + * Created only for non-special types (see the constructor). A default + * constructed `beast::insight::Gauge` holds a null impl and every + * mutator is a no-op, so a special type's gauge is safe to assign to + * but publishes nothing. + */ + /** @{ */ + beast::insight::Gauge waitingGauge; + beast::insight::Gauge runningGauge; + beast::insight::Gauge deferredGauge; + /** @} */ + JobTypeData( JobTypeInfo const& info, beast::insight::Collector::ptr collector, @@ -45,10 +89,17 @@ struct JobTypeData { load_.setTargetLatency(info.getAverageLatency(), info.getPeakLatency()); + // Special types have limit_ == 0 and bypass the limit logic + // entirely, so their `deferred` is always 0. Excluding them here + // matches the existing dequeue/execute rule. if (!info.special()) { - dequeue = collector_->makeEvent(info.name() + "_q"); + dequeue = collector_->makeEvent(info.name() + kSuffixQueued); execute = collector_->makeEvent(info.name()); + + waitingGauge = collector_->makeGauge(info.name() + kSuffixWaiting); + runningGauge = collector_->makeGauge(info.name() + kSuffixRunning); + deferredGauge = collector_->makeGauge(info.name() + kSuffixDeferred); } } diff --git a/include/xrpl/core/PerfLog.h b/include/xrpl/core/PerfLog.h index f09665e2915..441a6f6c767 100644 --- a/include/xrpl/core/PerfLog.h +++ b/include/xrpl/core/PerfLog.h @@ -92,30 +92,41 @@ class PerfLog * Log queued job * * @param type Job type + * @param name Job name as given to JobQueue::addJob. Distinguishes the + * several producers that share one job type. May be empty. */ virtual void - jobQueue(JobType const type) = 0; + jobQueue(JobType const type, std::string const& name) = 0; /** * Log job executing * * @param type Job type + * @param name Job name as given to JobQueue::addJob. Distinguishes the + * several producers that share one job type. May be empty. * @param dur Duration enqueued in microseconds * @param startTime Time that execution began * @param instance JobQueue worker thread instance */ virtual void - jobStart(JobType const type, microseconds dur, steady_time_point startTime, int instance) = 0; + jobStart( + JobType const type, + std::string const& name, + microseconds dur, + steady_time_point startTime, + int instance) = 0; /** * Log job finishing * * @param type Job type + * @param name Job name as given to JobQueue::addJob. Distinguishes the + * several producers that share one job type. May be empty. * @param dur Duration running in microseconds * @param instance Jobqueue worker thread instance */ virtual void - jobFinish(JobType const type, microseconds dur, int instance) = 0; + jobFinish(JobType const type, std::string const& name, microseconds dur, int instance) = 0; /** * Render performance counters in Json diff --git a/include/xrpl/core/ServiceRegistry.h b/include/xrpl/core/ServiceRegistry.h index 592964134b0..3dca3c46358 100644 --- a/include/xrpl/core/ServiceRegistry.h +++ b/include/xrpl/core/ServiceRegistry.h @@ -24,6 +24,10 @@ class Manager; namespace perf { class PerfLog; } // namespace perf +namespace telemetry { +class Telemetry; +class MetricsRegistry; +} // namespace telemetry // This is temporary until we migrate all code to use ServiceRegistry. class Application; @@ -224,6 +228,16 @@ class ServiceRegistry virtual perf::PerfLog& getPerfLog() = 0; + virtual telemetry::Telemetry& + getTelemetry() = 0; + + /** + * Return the MetricsRegistry, or nullptr if telemetry is disabled. + * Used by PerfLog and other hot paths to record OTel metrics. + */ + virtual telemetry::MetricsRegistry* + getMetricsRegistry() = 0; + // Configuration and state [[nodiscard]] virtual bool isStopping() const = 0; diff --git a/include/xrpl/json/json_value.h b/include/xrpl/json/json_value.h index 47ad3ac1e0c..77c5f652108 100644 --- a/include/xrpl/json/json_value.h +++ b/include/xrpl/json/json_value.h @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -623,6 +624,12 @@ class ValueConstIterator : public ValueIteratorBase public: using size_t = unsigned int; using difference_type = int; + // std::iterator_traits needs value_type and iterator_category to classify + // this as a Cpp17InputIterator; without them it defaults to output-only, + // which breaks standard algorithms (e.g. std::all_of). The iterator walks a + // map both ways via ++/--, so it is bidirectional. + using value_type = Value; + using iterator_category = std::bidirectional_iterator_tag; using reference = Value const&; using pointer = Value const*; using SelfType = ValueConstIterator; @@ -687,6 +694,10 @@ class ValueIterator : public ValueIteratorBase public: using size_t = unsigned int; using difference_type = int; + // See ValueConstIterator: value_type and iterator_category are required for + // std::iterator_traits to treat this as a bidirectional iterator. + using value_type = Value; + using iterator_category = std::bidirectional_iterator_tag; using reference = Value&; using pointer = Value*; using SelfType = ValueIterator; diff --git a/include/xrpl/proto/xrpl.proto b/include/xrpl/proto/xrpl.proto index d49920201ed..c2b28c87dda 100644 --- a/include/xrpl/proto/xrpl.proto +++ b/include/xrpl/proto/xrpl.proto @@ -85,6 +85,30 @@ message TMPublicKey { // If you want to send an amount that is greater than any single address of yours // you must first combine coins from one address to another. +// Trace context for OpenTelemetry distributed tracing across nodes. +// Uses W3C Trace Context format internally. +// +// Field numbering note: this message is embedded as field 1001 on +// TMTransaction, TMProposeSet, and TMValidation. Field numbers >= 1000 +// are reserved for optional, observability-only additions that must not +// collide with protocol-semantic fields (which historically use 1-99). +// Older peers that do not understand field 1001 will simply ignore it +// per protobuf wire-format rules, preserving backwards compatibility. +// +// trace_state is reserved for future use (secure tracing pipeline, +// OpenTelemetryPlan/secure-OTel.md). It is currently neither populated +// on inject nor read on extract; consumers must not rely on it. +message TraceContext { + optional bytes trace_id = 1; // 16-byte trace identifier + optional bytes span_id = 2; // 8-byte parent span identifier + optional uint32 trace_flags = 3; // bit 0 = sampled + // TODO: trace_state is reserved for W3C tracestate vendor-specific + // key-value pairs but is not yet read or written by + // TraceContextPropagator. Wire it when cross-vendor trace + // propagation is needed. + optional string trace_state = 4; // RESERVED — see TraceContext header note +} + enum TransactionStatus { tsNEW = 1; // origin node did/could not validate tsCURRENT = 2; // scheduled to go in this ledger @@ -101,6 +125,9 @@ message TMTransaction { required TransactionStatus status = 2; optional uint64 receiveTimestamp = 3; optional bool deferred = 4; // not applied to open ledger + + // Optional trace context for OpenTelemetry distributed tracing + optional TraceContext trace_context = 1001; } message TMTransactions { @@ -149,6 +176,9 @@ message TMProposeSet { // Number of hops traveled optional uint32 hops = 12 [deprecated = true]; + + // Optional trace context for OpenTelemetry distributed tracing + optional TraceContext trace_context = 1001; } enum TxSetStatus { @@ -194,6 +224,9 @@ message TMValidation { // Number of hops traveled optional uint32 hops = 3 [deprecated = true]; + + // Optional trace context for OpenTelemetry distributed tracing + optional TraceContext trace_context = 1001; } // An array of Endpoint messages diff --git a/include/xrpl/server/NetworkOPs.h b/include/xrpl/server/NetworkOPs.h index d3d3bb9f70f..4839b5f1f31 100644 --- a/include/xrpl/server/NetworkOPs.h +++ b/include/xrpl/server/NetworkOPs.h @@ -103,6 +103,14 @@ class NetworkOPs : public InfoSub::Source [[nodiscard]] virtual OperatingMode getOperatingMode() const = 0; + /** + * Time spent in the current operating mode so far, in microseconds. + * Same value as `server_state_duration_us` in server_info; exposed as a + * lightweight accessor so metrics can read it without building the full + * server_info JSON on every collection tick. + */ + [[nodiscard]] virtual std::chrono::microseconds + getServerStateDurationUs() const = 0; [[nodiscard]] virtual std::string strOperatingMode(OperatingMode const mode, bool const admin = false) const = 0; [[nodiscard]] virtual std::string diff --git a/include/xrpl/telemetry/CoroAwareContextStorage.h b/include/xrpl/telemetry/CoroAwareContextStorage.h new file mode 100644 index 00000000000..d7c047e7527 --- /dev/null +++ b/include/xrpl/telemetry/CoroAwareContextStorage.h @@ -0,0 +1,132 @@ +#pragma once + +/** + * A coroutine-aware OpenTelemetry runtime-context storage. + * + * OpenTelemetry's default ThreadLocalContextStorage keeps the active-context + * stack in a plain static thread_local, so the ambient span does NOT follow a + * JobQueue::Coro across yield/resume — a scope pushed on one worker is stranded + * when the coroutine resumes on another. This storage instead keeps the stack + * in an xrpl::LocalValue, which JobQueue::Coro::resume() swaps in and out with + * the coroutine (see Coro.ipp). The active context therefore rides the + * coroutine: scopes held across yield are safe, and per-line log-trace + * correlation (Log.cpp reads RuntimeContext::GetCurrent()) is retained. + * + * Off a coroutine, LocalValue transparently provides a per-thread store, so + * behaviour is identical to the default thread-local storage. + * + * +-------------------------------------------------+ + * | CoroAwareContextStorage | + * | (opentelemetry RuntimeContextStorage) | + * +-------------------------------------------------+ + * | - stack_ : LocalValue> | + * +-------------------------------------------------+ + * | + GetCurrent() : Context | + * | + Attach(ctx) : unique_ptr | + * | + Detach(token): bool | + * +-------------------------------------------------+ + * | backed by (coro-aware) + * +------------------------+ + * | xrpl::LocalValue store | + * | (swapped by Coro) | + * +------------------------+ + * + * Install once at telemetry start via + * opentelemetry::context::RuntimeContext::SetRuntimeContextStorage(), BEFORE + * any span is created (SDK requirement). + * + * @note Thread-safety: each thread/coroutine sees its own LocalValue store, so + * the stack is never shared across threads — no locking needed. The storage + * object itself is stateless apart from the LocalValue handle. + * @note Known limitation: install once, before the first span; resetting the + * storage while spans exist is undefined behaviour (SDK). A span created on a + * raw worker thread and later moved onto a coroutine does not retro-attach to + * the coroutine's store — not a pattern here (spans are created inside their + * own coro/job body). + * + * Example 1 — install at telemetry start (primary use): + * @code + * using opentelemetry::context::RuntimeContext; + * RuntimeContext::SetRuntimeContextStorage( + * opentelemetry::nostd::shared_ptr< + * opentelemetry::context::RuntimeContextStorage>( + * new xrpl::telemetry::CoroAwareContextStorage())); + * @endcode + * + * Example 2 — a scope held across a coroutine yield stays correct: + * @code + * // Inside a JobQueue::Coro body: + * auto span = ScopedSpanGuard(TraceCategory::Rpc, "rpc", "process"); + * context.coro->yield(); // may resume on another worker + * // span is still the ambient context here — the storage rode the coro. + * @endcode + * + * Example 3 — edge case: off a coroutine, behaves like thread-local storage: + * @code + * // On a plain worker thread (no coro): each thread has its own stack, + * // identical to opentelemetry's default ThreadLocalContextStorage. + * auto span = ScopedSpanGuard(TraceCategory::Ledger, "ledger", "build"); + * @endcode + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include + +#include +#include +#include + +#include + +namespace xrpl::telemetry { + +class CoroAwareContextStorage : public opentelemetry::context::RuntimeContextStorage +{ +public: + CoroAwareContextStorage() = default; + + // Non-copyable and non-movable: the LocalValue store is keyed by the + // address of stack_, so copying or moving would mis-key the store and + // strand its entries. Only ever new-ed once and installed on the SDK, so + // these are for intent/safety rather than a live bug. + CoroAwareContextStorage(CoroAwareContextStorage const&) = delete; + CoroAwareContextStorage& + operator=(CoroAwareContextStorage const&) = delete; + CoroAwareContextStorage(CoroAwareContextStorage&&) = delete; + CoroAwareContextStorage& + operator=(CoroAwareContextStorage&&) = delete; + + /** + * @return the current (top-of-stack) context for this coro/thread. + */ + opentelemetry::context::Context + GetCurrent() noexcept override; + + /** + * Push a context frame onto this coro/thread's stack. + * @param context the context to make current. + * @return a token that Detach() uses to pop back to the prior frame. + */ + opentelemetry::nostd::unique_ptr + Attach(opentelemetry::context::Context const& context) noexcept override; + + /** + * Pop the stack back through the frame the token refers to. + * @param token a token returned by Attach(). + * @return true if the frame was found and detached. + */ + bool + Detach(opentelemetry::context::Token& token) noexcept override; + +private: + /** + * The active-context stack, stored coro-locally. LocalValue hands back the + * stack for whichever store (coro or thread) is currently installed. + */ + LocalValue> stack_; +}; + +} // namespace xrpl::telemetry + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/include/xrpl/telemetry/DeterministicIdGenerator.h b/include/xrpl/telemetry/DeterministicIdGenerator.h new file mode 100644 index 00000000000..60d4aa5c4ac --- /dev/null +++ b/include/xrpl/telemetry/DeterministicIdGenerator.h @@ -0,0 +1,188 @@ +#pragma once + +#ifdef XRPL_ENABLE_TELEMETRY + +#include +#include +#include + +#include +#include +#include + +namespace xrpl::telemetry { + +/** + * OTel IdGenerator that can mint a deterministic (hash-derived) trace_id. + * + * By default the OTel SDK generates random trace_ids, so a span derived from + * a stable hash (e.g. a transaction id) cannot become a real trace root: the + * SDK either inherits the active span as parent or invents a random root id. + * This generator lets a caller pin the trace_id of the next forced-root span + * to a chosen 16-byte value, so hash-derived spans line up across nodes into + * one trace. When no value is pinned it behaves exactly like the default + * random generator. + * + * The pinned value lives in a thread-local slot set by PendingTraceId (below). + * Only GenerateTraceId() consults that slot, and only on the SDK's no-parent + * (root) branch; span_ids are always random. is_random_ is false so the W3C + * random-trace-id flag is not set on these deterministic ids. + * + * Dependency / data-flow diagram: + * + * +-----------------------------------------------------------+ + * | DeterministicIdGenerator | + * | (IdGenerator) | + * +-----------------------------------------------------------+ + * | - random_ : unique_ptr (random delegate) | + * +-----------------------------------------------------------+ + * | GenerateTraceId(): | + * | thread-local pending id set? --yes--> return that id | + * | --no --> random_ ... | + * | GenerateSpanId(): always ---------------> random_ ... | + * +-----------------------------------------------------------+ + * ^ | + * sets/clears| delegates| + * | v + * +----------------+ +--------------------+ + * | PendingTraceId | | RandomIdGenerator | + * | (RAII guard) | | (delegate) | + * +----------------+ +--------------------+ + * + * @note Thread safety: the pending trace_id is a file-local thread_local, so + * each thread sees only its own pinned value and no synchronization is needed. + * The pending id is consumed ONLY by GenerateTraceId(), which the SDK calls + * solely when a span has no valid parent; GenerateSpanId() never reads it and + * is always random. + * @note Limitation: minting a deterministic root only works when the caller + * forces the SDK's root branch (e.g. by starting the span with a + * Context{kIsRootSpanKey, true}); otherwise the SDK reuses the parent's + * trace_id and never calls GenerateTraceId(). The primary such caller is + * SpanGuard::hashSpan(), which forces the root branch to mint deterministic + * per-object trace roots. + * + * Example 1 - primary use, inside a forced-root hash span (shown as + * pseudocode): + * @code + * // std::array id = deriveTraceIdFromHash(txHash); + * // PendingTraceId const pending{id}; // pin id for this thread + * // auto root = Context{kIsRootSpanKey, true}; // force the no-parent branch + * // auto guard = telemetry.startSpan("tx.process", root); + * // // GenerateTraceId() returns `id`; the guard's trace_id == id. + * @endcode + * + * Example 2 - edge case, a normal child span never consults the pending id: + * @code + * // With an active parent span, startSpan() inherits the parent's trace_id + * // and the SDK does NOT call GenerateTraceId(), so no PendingTraceId is used. + * // auto child = parentGuard.childSpan(rpc_span::op::process); // random/parent trace_id + * @endcode + */ +class DeterministicIdGenerator final : public opentelemetry::sdk::trace::IdGenerator +{ + /** + * Random generator the deterministic path falls back to. Used for every + * span_id and for any trace_id when no PendingTraceId is active. + */ + std::unique_ptr random_; + +public: + /** + * Build a generator with is_random_ = false and a random delegate. + */ + DeterministicIdGenerator(); + + /** + * @return the thread's pending trace_id if a PendingTraceId is active, + * otherwise a fresh random trace_id from the delegate. Consuming the + * pending id clears it so it applies to exactly one root span. + */ + opentelemetry::trace::TraceId + GenerateTraceId() noexcept override; + + /** + * @return a fresh random span_id. Never uses the pending trace_id. + */ + opentelemetry::trace::SpanId + GenerateSpanId() noexcept override; +}; + +/** + * RAII guard that pins a deterministic trace_id for the next forced-root span + * started on this thread. + * + * Mirrors DiscardScope: it sets a thread-local pending trace_id on + * construction and clears it on destruction, so the pinned id stays confined + * to the guard's scope and cannot leak onto a later span. On destruction it + * asserts that the id was actually consumed by GenerateTraceId() — if it was + * not, the SDK took a branch other than the root branch the caller intended, + * which is a bug worth catching in debug/test builds. + * + * Wrap ONLY the single forced-root startSpan() call that must receive the + * deterministic id. Non-copyable and non-movable: its sole purpose is the + * scoped lifetime of the pending id. + * + * @note Thread safety: the pending id is thread-local, so a guard on one + * thread never affects another. Construct and destroy the guard on the same + * thread as the startSpan() call it wraps. + * @note Limitation: the consumed-assert only holds when the wrapped span is + * started on the SDK root branch (Context{kIsRootSpanKey, true}); wrapping a + * child span would leave the id unconsumed and trip the assert. Nesting two + * guards on one thread is unsupported: the inner guard consumes/clears first, + * so the outer id is silently dropped and its destructor trips the assert. + * + * Example 1 - primary use, wrapping a forced-root span start (pseudocode): + * @code + * // { + * // PendingTraceId const pending{id}; // pin for this scope + * // auto root = Context{kIsRootSpanKey, true}; + * // auto guard = telemetry.startSpan(name, root);// consumes the pinned id + * // } // ~PendingTraceId asserts the id was consumed, then clears it + * @endcode + * + * Example 2 - edge case, misuse detection: wrapping a non-root span leaves the + * id unconsumed, so ~PendingTraceId trips XRPL_ASSERT in debug/test builds. + */ +class PendingTraceId +{ +public: + /** + * Pin @p id as the pending trace_id for this thread. + * @param id The 16-byte trace_id the next forced-root span should adopt. + */ + explicit PendingTraceId(std::array const& id) noexcept; + + /** + * Assert the pinned id was consumed by a root span, then clear it so it + * never leaks onto the next span (even in release, where the assert is a + * no-op). + */ + ~PendingTraceId() noexcept; + + PendingTraceId(PendingTraceId const&) = delete; + PendingTraceId& + operator=(PendingTraceId const&) = delete; + PendingTraceId(PendingTraceId&&) = delete; + PendingTraceId& + operator=(PendingTraceId&&) = delete; +}; + +/** + * Running count of deterministic trace_ids that were pinned by a + * PendingTraceId but never consumed by a forced-root span before the guard was + * destroyed. Bumped once per silent drop. + * + * ~PendingTraceId also asserts on this condition, but the assert is a no-op in + * release builds, so this monotonic counter is the only signal that a + * deterministic trace root was lost there. It is process-wide (summed across + * all threads) and intended for a future diagnostic metric or debugger + * inspection; reading it has no side effects. + * + * @return the total number of unconsumed deterministic trace_id drops so far. + */ +[[nodiscard]] std::uint64_t +unconsumedDeterministicIdDrops() noexcept; + +} // namespace xrpl::telemetry + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/include/xrpl/telemetry/DiscardFlag.h b/include/xrpl/telemetry/DiscardFlag.h new file mode 100644 index 00000000000..1bd1f668bb8 --- /dev/null +++ b/include/xrpl/telemetry/DiscardFlag.h @@ -0,0 +1,95 @@ +#pragma once + +/** + * Thread-local discard signaling between SpanGuard and the span processor. + * + * SpanGuard::discard() wants to drop a span without sending it to the + * exporter. The OTel SDK calls SpanProcessor::OnEnd() synchronously on the + * same thread that calls Span::End(), so a thread-local flag set just before + * End() and read inside OnEnd() lets FilteringSpanProcessor drop the span + * before it enters the batch export queue. + * + * This side-channel avoids inspecting the Recordable's internals (which vary + * by exporter type — SpanData vs OtlpRecordable). + * + * The flag is a *private* thread-local member of DiscardScope, mutated only by + * its constructor and destructor. This gives real access control rather than a + * naming convention: no code that includes this header can flip the flag + * directly — it can only enter a DiscardScope (which sets and clears the flag + * over its own lifetime) and observe the state via DiscardScope::isActive(). + * Binding set/clear to a scope also means the flag cannot leak onto the next + * span even if End() were to throw. + * + * Kept in a separate header to avoid transitive include bloat: SpanGuard.h + * only needs this signaling, not the full Telemetry.h with BasicConfig/Journal. + * + * Usage: + * @code + * // In SpanGuard::discard(): + * { + * DiscardScope discardScope; // flag set for this scope only + * span->End(); // OnEnd() runs synchronously, sees flag + * } // flag cleared here, unconditionally + * + * // In FilteringSpanProcessor::OnEnd(): + * if (DiscardScope::isActive()) + * return; // drop the span + * @endcode + * + * @note Thread safety: the flag is thread-local, so each thread observes only + * its own discard signal — no synchronization is required. + * + * @see SpanGuard::discard(), FilteringSpanProcessor (Telemetry.cpp) + */ + +namespace xrpl::telemetry { + +/** + * RAII guard that marks the current thread's span for discard. + * + * Sets the thread-local discard flag on construction and clears it on + * destruction, so a span ended within the guard's scope is dropped by + * FilteringSpanProcessor::OnEnd() while the flag stays confined to that scope. + * The flag is private and mutated only here, so no other code can set it. + * Non-copyable and non-movable — its sole purpose is the scoped flag lifetime. + */ +class DiscardScope +{ +public: + DiscardScope() noexcept + { + discarding = true; + } + + ~DiscardScope() + { + discarding = false; + } + + DiscardScope(DiscardScope const&) = delete; + DiscardScope& + operator=(DiscardScope const&) = delete; + DiscardScope(DiscardScope&&) = delete; + DiscardScope& + operator=(DiscardScope&&) = delete; + + /** + * @return true if the current thread is inside a DiscardScope, i.e. the + * span ending now should be dropped rather than exported. Read by + * FilteringSpanProcessor::OnEnd(). + */ + [[nodiscard]] static bool + isActive() noexcept + { + return discarding; + } + +private: + /** + * Thread-local discard flag. Private, so only this class's ctor/dtor can + * mutate it; observers use isActive(). One instance per thread. + */ + inline static thread_local bool discarding = false; +}; + +} // namespace xrpl::telemetry diff --git a/include/xrpl/telemetry/GetObjectMetricNames.h b/include/xrpl/telemetry/GetObjectMetricNames.h new file mode 100644 index 00000000000..e2951a43d06 --- /dev/null +++ b/include/xrpl/telemetry/GetObjectMetricNames.h @@ -0,0 +1,160 @@ +#pragma once + +// cspell:ignore ISTOGRAM +// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's +// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. + +/** + * Metric names, label keys, label values, and descriptions for the + * `TMGetObjectByHash` request path. + * + * These constants are shared across two modules, which is why they live in a + * header rather than in either translation unit's unnamed namespace: + * + * GetObjectMetricNames.h + * | + * +--> PeerImp.cpp (XRPL_METRIC_* call sites: records the + * | instruments) + * | + * +--> MetricsRegistry.cpp (addMicrosecondHistogramView: registers + * the explicit bucket boundaries for + * kGetObjectLookupUs) + * + * `kGetObjectLookupUs` in particular is referenced from both sites. A + * copy-pasted literal would let the two drift, and a drifted name silently + * drops the bucket override -- the histogram would fall back to the SDK + * default boundaries, which top out at 10,000 (10 ms), so every quantile + * would saturate. This mirrors the reason the existing + * `kJobQueuedDurationUs` / `kJobRunningDurationUs` / `kRpcMethodDurationUs` + * constants exist in MetricsRegistry.cpp; those three are referenced only + * within that one file, so they stay file-local. + * + * Placed under `include/xrpl/telemetry/` because the two consumers sit in + * different levelization modules: PeerImp.cpp is `xrpld.overlay` and + * MetricsRegistry.cpp is `xrpld.telemetry`. Both are allowed to depend on + * `xrpl.telemetry` (see `xrpld.overlay > xrpl.telemetry` and + * `xrpld.telemetry > xrpl.telemetry` in the levelization ordering results), so + * `include/xrpl/` is the one level both can reach. Keeping the constants in + * `src/xrpld/telemetry/` would have required overlay to include a private + * xrpld header from another module, flipping a levelization edge. Both sites + * include this file as ``. + * + * Example usage -- recording a histogram: + * @code + * XRPL_METRIC_HISTOGRAM_RECORD( + * app_, kGetObjectRequestObjects, kGetObjectRequestObjectsDesc, requested); + * @endcode + * + * Example usage -- edge case: the same instrument recorded under two + * different label values, which is why the label key and both values are + * constants rather than literals: + * @code + * XRPL_METRIC_COUNTER_ADD_LABELED( + * app_, kGetObjectLookupsTotal, kGetObjectLookupsTotalDesc, hits, + * {{kLabelResult, std::string(kResultHit)}}); + * XRPL_METRIC_COUNTER_ADD_LABELED( + * app_, kGetObjectLookupsTotal, kGetObjectLookupsTotalDesc, misses, + * {{kLabelResult, std::string(kResultMiss)}}); + * @endcode + * + * @note These are `constexpr char[]`, not `constexpr std::string_view`. The + * OTel C++ API takes `nostd::string_view`, which on this build is OTel's own + * type (the build defines only OPENTELEMETRY_ABI_VERSION_NO, not + * OPENTELEMETRY_STL_VERSION, so `nostd::string_view` is not an alias for + * `std::string_view`). It converts from `char const*` and from + * `std::string`, but has no converting constructor from `std::string_view`, + * so a `string_view` constant would not compile at the call sites. This also + * matches the existing convention in MetricsRegistry.cpp. + * + * @note Header-only constants with no runtime state, so there is nothing to + * synchronize -- safe to include from any thread context. + */ + +namespace xrpl::telemetry { + +// ===== Metric names ========================================================== + +/** + * Requests refused by `onMessage(TMGetObjectByHash)` before any NodeStore + * access, split by the `reason` label. + */ +inline constexpr char kGetObjectRejectedTotal[] = "getobject_rejected_total"; + +/** + * Distribution of the object count requested per message. + */ +inline constexpr char kGetObjectRequestObjects[] = "getobject_request_objects"; + +/** + * Wall time spent in the NodeStore fetch loop, in microseconds. + * + * Referenced twice: here at the record site, and by + * `addMicrosecondHistogramView()` in MetricsRegistry.cpp. Both must use this + * one constant. + */ +inline constexpr char kGetObjectLookupUs[] = "getobject_lookup_us"; + +/** + * NodeStore lookups performed, split by the `result` label. + */ +inline constexpr char kGetObjectLookupsTotal[] = "getobject_lookups_total"; + +/** + * Distribution of the dynamic resource charge applied per request. + */ +inline constexpr char kGetObjectCharge[] = "getobject_charge"; + +// ===== Label keys ============================================================ + +/** + * Label key distinguishing a hit from a miss on `kGetObjectLookupsTotal`. + */ +inline constexpr char kLabelResult[] = "result"; + +/** + * Label key naming which gate refused the request. + */ +inline constexpr char kLabelReason[] = "reason"; + +// ===== Label values ========================================================== + +/** + * `kLabelResult` value: the object was found in the NodeStore. + */ +inline constexpr char kResultHit[] = "hit"; + +/** + * `kLabelResult` value: the object was not found. + */ +inline constexpr char kResultMiss[] = "miss"; + +/** + * `kLabelReason` value: more objects requested than the handler accepts. + */ +inline constexpr char kReasonOversize[] = "oversize"; + +/** + * `kLabelReason` value: the ledger hash was not uint256-sized. + */ +inline constexpr char kReasonMalformedLedgerHash[] = "malformed_ledgerhash"; + +// ===== Instrument descriptions =============================================== + +/** @{ */ +inline constexpr char kGetObjectRejectedTotalDesc[] = + "TMGetObjectByHash requests refused before any NodeStore access, by reason"; + +inline constexpr char kGetObjectRequestObjectsDesc[] = + "Objects requested per TMGetObjectByHash message"; + +inline constexpr char kGetObjectLookupUsDesc[] = + "Time spent in the TMGetObjectByHash NodeStore fetch loop, in microseconds"; + +inline constexpr char kGetObjectLookupsTotalDesc[] = + "TMGetObjectByHash NodeStore lookups, by hit/miss result"; + +inline constexpr char kGetObjectChargeDesc[] = + "Dynamic resource charge applied per TMGetObjectByHash request"; +/** @} */ + +} // namespace xrpl::telemetry diff --git a/include/xrpl/telemetry/Redaction.h b/include/xrpl/telemetry/Redaction.h new file mode 100644 index 00000000000..3b1e8f67ef9 --- /dev/null +++ b/include/xrpl/telemetry/Redaction.h @@ -0,0 +1,66 @@ +#pragma once + +/** + * Account-address redaction for telemetry span attributes. + * + * Path-finding RPC handlers would otherwise emit the caller's raw + * account addresses as span attributes. To keep plaintext addresses out + * of the telemetry backend, they are hashed at the point of emission. + * This header exposes a single pure helper that turns an address into a + * short, stable, obfuscated token. + * + * Data flow: + * + * handler -> redactAccount(addr) -> span attribute -> OTLP export + * + * The returned token is the first 16 hex characters (lowercase) of the + * SHA-512Half digest of the address. It is deterministic (same address + * always maps to the same token) so operators can still correlate spans + * for a given account across nodes and restarts. + * + * The hash is unsalted, so it is obfuscation, not a secrecy guarantee: + * XRP account addresses are a public, enumerable set, so a determined + * observer with the telemetry stream could rebuild the address->token + * mapping. The goal here is to keep plaintext addresses out of traces + * and dashboards, not to defend against a precomputation attack. A salt + * is intentionally omitted because it would break cross-node/restart + * correlation, which is the reason for hashing rather than dropping. + * + * A second, independent hashing layer runs in the OpenTelemetry + * Collector (an `attributes/hash` processor) as defense-in-depth for + * any node that emits a raw value. + * + * @note This function is pure and reentrant: it holds no global state, + * performs no I/O, and is safe to call concurrently from any thread. + * + * Usage example: + * @code + * #include + * using namespace xrpl::telemetry; + * + * span.setAttribute( + * pathfind_span::attr::sourceAccount, redactAccount(src.asString())); + * @endcode + * + * Edge case (empty input yields empty output): + * @code + * assert(redactAccount("") == ""); + * @endcode + */ + +#include +#include + +namespace xrpl::telemetry { + +/** + * Hash an account address into a short, stable, obfuscated token. + * + * @param addr The account address to redact (e.g. an r-address). + * @return The first 16 lowercase hex characters of sha512Half(addr), + * or an empty string when @p addr is empty. + */ +[[nodiscard]] std::string +redactAccount(std::string_view addr); + +} // namespace xrpl::telemetry diff --git a/include/xrpl/telemetry/SpanGuard.h b/include/xrpl/telemetry/SpanGuard.h new file mode 100644 index 00000000000..4758602b747 --- /dev/null +++ b/include/xrpl/telemetry/SpanGuard.h @@ -0,0 +1,1254 @@ +#pragma once + +/** + * RAII guards for OpenTelemetry trace spans. + * + * This header declares two complementary span guards, split by + * responsibility: + * + * - SpanGuard — owns a span only. Thread-free: it never touches the + * OTel thread-local context stack, so it may be moved + * to and destroyed on any thread. Movable AND + * move-assignable. + * - ScopedSpanGuard — owns a SpanGuard plus an active OTel Scope that + * pushes the span onto the constructing context store's + * stack. Store-bound: it must be created and destroyed + * while the same LocalValue context store is active (the + * same thread, or the same JobQueue coroutine even if it + * resumes on another worker). Non-copyable and + * non-movable. + * + * Both wrap all OpenTelemetry types behind the pimpl idiom so no + * opentelemetry headers are exposed here. When XRPL_ENABLE_TELEMETRY + * is not defined, both are empty classes with all-inline no-op methods + * — zero overhead, zero dependencies. + * + * SpanGuard dependency diagram: + * + * +--------------------------------------------+ + * | SpanGuard | + * | (unscoped, thread-free) | + * +--------------------------------------------+ + * | - impl_ : unique_ptr (pimpl) | + * +--------------------------------------------+ + * | + span(cat, prefix, name) [static] | + * | + freshRoot(cat, prefix, name) [static] | + * | + childSpan(name) : SpanGuard | + * | + linkedSpan(name) : SpanGuard | + * | + spanContext() : SpanContext | + * | + threadLocalContext() [static] | + * | + setAttribute(key, value) | + * | + setOk() / setError(desc) | + * | + addEvent(name) | + * | + recordException(e) | + * | + discard() | + * | + activate() : ScopedActivation | + * | + operator bool() | + * +--------------------------------------------+ + * | hides (pimpl) + * +--------+ + * | Span | + * | (OTel) | + * +--------+ + * + * Static factory methods access the global Telemetry instance + * internally (via Telemetry::getInstance()), check whether tracing + * is enabled for the requested subsystem, and return either an + * active guard or a null (no-op) guard. Callers never need a + * Telemetry reference. + * + * Usage examples: + * + * Span names and attribute keys come from per-module `*SpanNames.h` + * headers (e.g. RpcSpanNames.h, TxSpanNames.h) as typed compile-time + * constants — never raw string literals — so the naming spec is + * enforced at the call site and dashboards stay in sync. + * + * 1. Basic RPC tracing (factory method with category): + * @code + * #include + * using namespace xrpl::telemetry; + * + * auto span = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::command, commandName); + * span.setAttribute(rpc_span::attr::command, commandName); + * span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::success); + * // span ended automatically when the guard is destroyed + * @endcode + * + * 2. Error recording: + * @code + * auto span = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::command, commandName); + * try { + * doWork(); + * span.setOk(); + * } catch (std::exception const& e) { + * span.recordException(e); + * } + * @endcode + * + * 3. Cross-thread context propagation: + * @code + * #include + * using namespace xrpl::telemetry; + * + * // Thread A: create span and capture its own context as parent + * auto span = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::process); + * auto ctx = span.spanContext(); + * + * // Thread B: create child with captured context + * auto child = SpanGuard::childSpan(rpc_span::op::process, ctx); + * @endcode + * + * 4. Conditional check (rarely needed — methods are no-ops on null): + * @code + * auto span = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::httpRequest); + * if (span) { + * // expensive attribute computation only when active + * span.setAttribute(rpc_span::attr::requestPayloadSize, computeSize()); + * } + * @endcode + * + * 5. Tail-based filtering via discard(): + * @code + * auto span = SpanGuard::span( + * TraceCategory::Transactions, tx_span::prefix::tx, tx_span::op::process); + * auto result = preflight(tx); + * if (result != tesSUCCESS) { + * span.discard(); // drop span, never exported + * return result; + * } + * @endcode + * + * 6. Fresh trace root at an inbound entry point (primary freshRoot use): + * @code + * #include + * using namespace xrpl::telemetry; + * + * // A peer message handled on a shared worker thread that may + * // already have unrelated spans active — start a clean root so + * // those do not become parents of this trace. Names come from a + * // *SpanNames.h header, never raw literals. + * auto span = SpanGuard::freshRoot( + * TraceCategory::Peer, seg::peer, peer_span::op::validationReceive); + * span.setAttribute(peer_span::attr::ledgerHash, hashStr); + * @endcode + * + * 7. Hand a span to a job on another thread (thread-free — just move): + * @code + * #include + * using namespace xrpl::telemetry; + * + * // SpanGuard owns no thread-local Scope, so it is safe to move + * // into a job and end on the worker thread. No detach step is + * // needed — just move the guard in. + * auto span = SpanGuard::span( + * TraceCategory::Ledger, seg::ledger, ledger_span::op::build); + * jobQueue.addJob( + * [g = std::move(span)]() mutable { + * doWork(); + * // g's span ends when the job's lambda is destroyed, + * // on the worker thread. + * }); + * @endcode + * + * @note Thread safety: SpanGuard is thread-free. It holds only the + * span (no Scope), so it never binds to a thread-local context stack + * and may be moved to and destroyed on any thread. To make a span the + * ambient parent for child spans on a thread, wrap it in a + * ScopedSpanGuard. Use spanContext() to propagate this span's own + * context across threads explicitly, or threadLocalContext() to + * snapshot the thread's currently-active context. + * + * @note Move semantics: Move construction and move assignment both + * transfer ownership of the pimpl pointer. There is no Scope to + * re-bind, so move assignment is safe and enabled. Copy is deleted. + * + * @note Known limitations: + * - Attributes cannot be removed per the OTel spec; use + * setAttribute with an empty value as a convention. + * - The raw OTel Span is intentionally not exposed — all interaction + * goes through the public methods. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace protocol { +class TraceContext; +} // namespace protocol + +namespace xrpl::telemetry { + +/** + * Trace subsystem categories for conditional span creation. + * + * Each value maps to a runtime config flag (e.g. `trace_rpc=1`). + * Used by SpanGuard::span(TraceCategory, prefix, name) to decide + * whether to create a real span or return a null guard. + */ +enum class TraceCategory { Rpc, Transactions, Consensus, Peer, Ledger }; + +/** + * Raw trace context bytes for cross-node propagation. + * + * Holds the binary trace_id, span_id, and trace_flags extracted from + * an active span. Used by protocol-layer code to inject trace context + * into outgoing protobuf messages without depending on OTel types. + * + * @see SpanGuard::getTraceBytes(), TraceContextPropagator.h + */ +struct TraceBytes +{ + std::array traceId{}; ///< 16-byte W3C trace identifier. + std::array spanId{}; ///< 8-byte span id of current span. + std::uint8_t traceFlags{0}; ///< W3C trace flags (bit 0 = sampled). + bool valid{false}; ///< True if this struct holds data from an active span. +}; + +/** + * Key-value pair for span event attributes. + * Used by addEvent(name, attrs) to attach structured metadata to events. + */ +using EventAttribute = std::pair; + +/** + * Opaque wrapper for an OTel context snapshot. + * + * Used to propagate trace context across threads. Created by + * SpanGuard::spanContext() (the guard's own span) or + * SpanGuard::threadLocalContext() (the thread's current context), + * consumed by SpanGuard::childSpan() or SpanGuard::linkedSpan() + * with an explicit parent/link context. + */ +class SpanContext +{ + friend class SpanGuard; + +#ifdef XRPL_ENABLE_TELEMETRY + struct Impl; + std::shared_ptr impl_; + explicit SpanContext(std::shared_ptr impl); +#endif + +public: + SpanContext() = default; + + /** + * @return true if this context holds a valid trace context. + */ +#ifdef XRPL_ENABLE_TELEMETRY + [[nodiscard]] bool + isValid() const noexcept; +#else + // NOLINTBEGIN(readability-convert-member-functions-to-static) + [[nodiscard]] bool + isValid() const noexcept + { + return false; + } + // NOLINTEND(readability-convert-member-functions-to-static) +#endif +}; + +// --------------------------------------------------------------------------- +// Real implementation (pimpl, compiled in SpanGuard.cpp) +// --------------------------------------------------------------------------- +#ifdef XRPL_ENABLE_TELEMETRY + +// SpanGuard::activate() returns a ScopedActivation by value; the full class is +// defined after ScopedSpanGuard below. Forward-declared here so the return type +// is named before its definition. +class ScopedActivation; + +/** + * RAII owner of an OTel span (unscoped, thread-free). + * + * Holds only the span behind the Impl pointer — it never pushes the + * span onto the thread-local context stack, so it carries no + * thread-affinity and may be moved to and destroyed on any thread. + * The span is ended when the guard is destroyed (unless discard() + * ended it earlier). Non-copyable; movable and move-assignable. + * + * To activate a span as the ambient parent for child spans on a + * thread, wrap the guard in a ScopedSpanGuard. + */ +class SpanGuard +{ + struct Impl; + std::unique_ptr impl_; + + explicit SpanGuard(std::unique_ptr impl) noexcept; + + // ScopedSpanGuard wraps a SpanGuard and needs the raw span to build + // its Scope; it reads impl_ directly so no OTel type leaks here. + friend class ScopedSpanGuard; + +public: + /** + * Construct a null (no-op) guard. All methods are safe to call. + */ + SpanGuard(); + ~SpanGuard(); + + SpanGuard(SpanGuard&& other) noexcept; + SpanGuard& + operator=(SpanGuard&&) noexcept; + SpanGuard(SpanGuard const&) = delete; + SpanGuard& + operator=(SpanGuard const&) = delete; + + // --- Static factory methods ---------------------------------------- + + /** + * Create a span guarded by a TraceCategory flag. + * The span name is built as "prefix.name". Returns a null guard + * if the category is disabled in config. + * @param cat Trace subsystem category. + * @param prefix Span name prefix (e.g. "rpc.command"). + * @param name Span name suffix (e.g. "submit"). + */ + [[nodiscard]] static SpanGuard + span(TraceCategory cat, std::string_view prefix, std::string_view name) noexcept; + + /** + * Create a span that always starts a fresh trace root. + * + * Like span(), but ignores this thread's active span so the new + * span never inherits an ambient parent. Use at an inbound entry + * point (e.g. a peer message received on a shared worker thread) + * so unrelated work already on the stack does not pollute the + * trace. The span name is built as "prefix.name". Returns a null + * guard if the category is disabled in config. + * + * @param cat Trace subsystem category. + * @param prefix Span name prefix (e.g. "peer"). + * @param name Span name suffix (e.g. "validation.receive"). + * @return An active root-span guard, or a null guard if disabled. + */ + [[nodiscard]] static SpanGuard + freshRoot(TraceCategory cat, std::string_view prefix, std::string_view name) noexcept; + + // --- Child / linked span creation ---------------------------------- + + /** + * Create a child span parented to the CURRENT ambient context of this + * store, not to this guard's own span. It reads the active context via the + * runtime context store, so it is meaningful only when a scope is active on + * THIS store — a ScopedSpanGuard, a ScopedActivation, or this guard's span + * activated some other way. With coroutine-aware storage the ambient + * context is the correct parent in scoped and coroutine contexts (a + * coroutine keeps its store across a resume on another worker). When no + * scope is active the child inherits whatever context happens to be on the + * store, which may be unrelated. + * + * For explicit, store-independent parenting to a specific span (e.g. + * across stores or threads), capture spanContext() and use + * childSpan(name, ctx) instead. + * @param name Span name for the child. + * @return A new guard, or null if this guard is inactive. + */ + [[nodiscard]] SpanGuard + childSpan(std::string_view name) const noexcept; + + /** + * Create a child span parented to an explicit captured context. + * @param name Span name for the child. + * @param parentCtx Context captured via spanContext(). + * @return A new guard, or null if parentCtx is invalid. + */ + [[nodiscard]] static SpanGuard + childSpan(std::string_view name, SpanContext const& parentCtx) noexcept; + + /** + * Create a span linked (follows-from) to this guard's span. + * The new span is NOT a child — it starts a new sub-tree but + * carries a causal link to this span. + * @param name Span name for the linked span. + * @return A new guard, or null if this guard is inactive. + */ + [[nodiscard]] SpanGuard + linkedSpan(std::string_view name) const noexcept; + + /** + * Create a span linked to an explicit captured context. + * @param name Span name for the linked span. + * @param linkCtx Context to link from. + * @return A new guard, or null if linkCtx is invalid. + */ + [[nodiscard]] static SpanGuard + linkedSpan(std::string_view name, SpanContext const& linkCtx) noexcept; + + // --- Hash-derived span (category-gated) ----------------------------- + + /** + * Create a span whose trace_id is derived from arbitrary hash data. + * trace_id = hashData[0:16], span_id = random. Gated by the given + * TraceCategory. All nodes using the same hash independently produce + * spans under the same trace_id, enabling cross-node correlation + * without context propagation. + * @param cat Trace subsystem category. + * @param name Full span name (e.g. "tx.receive"). + * @param hashData Pointer to at least 16 bytes of hash data. + * @param hashSize Size of the hash buffer (must be >= 16). + * @param followsFrom Optional captured context to attach as a + * follows-from link. Use to stitch sequential + * top-level spans (e.g. consecutive consensus + * rounds). Ignored if nullptr or invalid. + */ + static SpanGuard + hashSpan( + TraceCategory const cat, + std::string_view const name, + std::uint8_t const* const hashData, + std::size_t const hashSize, + SpanContext const* followsFrom = nullptr); + + /** + * Create a hash-derived span with a remote parent. + * trace_id = hashData[0:16], parent span_id from protobuf context + * propagation. Produces a child span of the sender's span while + * sharing the deterministic trace_id. + * @param cat Trace subsystem category. + * @param name Full span name. + * @param hashData Pointer to at least 16 bytes of hash data. + * @param hashSize Size of the hash buffer (must be >= 16). + * @param parentSpanId Pointer to 8 bytes of parent span ID. + * @param parentSpanSize Size of parent span ID buffer (must be 8). + * @param traceFlags Trace flags from remote context. + */ + static SpanGuard + hashSpan( + TraceCategory const cat, + std::string_view const name, + std::uint8_t const* const hashData, + std::size_t const hashSize, + std::uint8_t const* const parentSpanId, + std::size_t const parentSpanSize, + std::uint8_t const traceFlags); + + // --- Context capture ----------------------------------------------- + + /** + * Return a SpanContext referring to THIS guard's own span, for use + * as an explicit parent in childSpan(name, ctx) across threads. + * Independent of the thread's active span (a SpanGuard is unscoped). + * Returns an invalid context if this guard is null. + * @return A SpanContext holding this guard's own span. + * @note This is NOT the thread's ambient context — use + * threadLocalContext() for that. + */ + [[nodiscard]] SpanContext + spanContext() const noexcept; + + /** + * Snapshot the calling thread's CURRENTLY-ACTIVE OTel context (the + * ambient span on this thread's context stack), for cross-thread + * propagation. This is the OpenTelemetry "capture current context" + * operation: capture -> pass to another thread -> childSpan(name, + * ctx). Unlike spanContext() (which refers to a specific guard's own + * span), this reflects whatever span is active on THIS thread right + * now, or an invalid context if none is active. + * @return A SpanContext holding the thread's current context. + */ + [[nodiscard]] static SpanContext + threadLocalContext() noexcept; + + /** + * Extract raw trace context bytes from this span for propagation. + * + * Unlike threadLocalContext() which captures the thread-local runtime + * context, this method reads the span's own SpanContext directly. + * Safe to call from any thread that holds a reference to this guard. + * + * @return A TraceBytes struct with valid=true if the span is active + * and has a valid context, or valid=false otherwise. + */ + [[nodiscard]] TraceBytes + getTraceBytes() const; + + /** + * Inject the calling thread's currently-active OTel context into a + * protobuf TraceContext message for cross-node propagation. + * + * Encapsulates `RuntimeContext::GetCurrent()` + `injectToProtobuf` + * so callers in app-layer code (e.g. RCLConsensus broadcasting + * TMProposeSet / TMValidation) don't depend on any OTel headers. + * No-op if telemetry is disabled or no span is active. + * + * @param proto The protobuf TraceContext to populate. + */ + static void + injectCurrentContextToProtobuf(protocol::TraceContext& proto); + + // --- Attribute setters (explicit overloads, no OTel types) --------- + + /** + * Set a string attribute. No-op on a null guard. + */ + void + setAttribute(std::string_view key, std::string_view value) noexcept; + + /** + * Set a string attribute (C-string overload). No-op on a null guard. + */ + void + setAttribute(std::string_view key, char const* value) noexcept; + + /** + * Set an integer attribute. No-op on a null guard. + */ + void + setAttribute(std::string_view key, std::int64_t value) noexcept; + + /** + * Set a floating-point attribute. No-op on a null guard. + */ + void + setAttribute(std::string_view key, double value) noexcept; + + /** + * Set a boolean attribute. No-op on a null guard. + */ + void + setAttribute(std::string_view key, bool value) noexcept; + + // --- Status / events ----------------------------------------------- + + /** + * Mark the span status as OK. No-op on a null guard. + */ + void + setOk() noexcept; + + /** + * Mark the span status as error. No-op on a null guard. + * @param description Optional human-readable error description. + */ + void + setError(std::string_view description = "") noexcept; + + /** + * Add a named event to the span's timeline. No-op on a null guard. + * @param name Event name. + */ + void + addEvent(std::string_view name) noexcept; + + /** + * Add a named event with key-value attributes to the span's timeline. + * No-op on a null guard. + * @param name Event name. + * @param attrs Attribute pairs (all string_view for simplicity). + */ + void + addEvent(std::string_view name, std::initializer_list attrs) noexcept; + + /** + * Record an exception as a span event following OTel semantic + * conventions, and mark the span status as error. + * No-op on a null guard. + * @param e The exception to record. + */ + void + recordException(std::exception const& e) noexcept; + + /** + * Mark this span for discard and end it immediately. + * The FilteringSpanProcessor drops the span before it enters the + * batch export queue. After discard(), the guard is inert. + */ + void + discard() noexcept; + + // --- Activation (non-owning) --------------------------------------- + + /** + * Activate this guard's span as the current context for the returned + * activation's lifetime, without transferring ownership. Returns a no-op + * activation if this guard is null. See ScopedActivation. + * @return an RAII activation; drop it to restore the prior context. + */ + [[nodiscard]] ScopedActivation + activate() const noexcept; + + /** + * @return true if this guard holds an active span. + */ + explicit + operator bool() const noexcept; +}; + +/** + * RAII guard that activates a span on the current context store (scoped). + * + * Wraps a SpanGuard (which owns the span) plus an OTel Scope that + * pushes the span onto the active context store's stack for the + * guard's lifetime, so child spans created under that store inherit + * it as parent. On destruction the Scope pops BEFORE the span ends + * (member order: guard first, scope second). Non-copyable and + * non-movable — factories return unnamed temporaries, so guaranteed + * copy elision (C++17) covers `auto s = ScopedSpanGuard::freshRoot(...)`. + * + * When the span must outlive the current store (e.g. handed to a job + * queue), convert to a plain SpanGuard with `operator SpanGuard() &&`: + * that pops the Scope eagerly under the constructing store and yields a + * thread-free guard. + * + * ScopedSpanGuard dependency diagram: + * + * +--------------------------------------------+ + * | ScopedSpanGuard | + * | (scoped, store-bound) | + * +--------------------------------------------+ + * | - impl_ : unique_ptr (pimpl) | + * +--------------------------------------------+ + * | + (cat, prefix, name) [ctor] | + * | + freshRoot(cat, prefix, name) [static] | + * | + childSpan(name) : ScopedSpanGuard | + * | + linkedSpan(name) : ScopedSpanGuard | + * | + operator SpanGuard() && (handoff) | + * | + setAttribute / setOk / setError / ... | + * | + spanContext() / discard() | + * | + operator bool() | + * +--------------------------------------------+ + * | hides (pimpl) + * +------+--------------------+ + * | | + * +----------+ +-----------------------------+ + * | SpanGuard| | optional | + * | (span) | | (OTel, store-bound; | + * | | | present : span active) | + * +----------+ +-----------------------------+ + * + * Usage examples: + * + * 1. Same-thread scoped tracing (child inherits parent automatically): + * @code + * using namespace xrpl::telemetry; + * + * ScopedSpanGuard span( + * TraceCategory::Rpc, rpc_span::prefix::command, commandName); + * span.setAttribute(rpc_span::attr::command, commandName); + * // childSpan parents to `span` because it is active on this thread + * auto child = span.childSpan(rpc_span::op::process); + * @endcode + * + * 2. Capture on this thread, hand off to another (edge case): + * @code + * using namespace xrpl::telemetry; + * + * auto scoped = ScopedSpanGuard::freshRoot( + * TraceCategory::Ledger, seg::ledger, ledger_span::op::build); + * // Pop the scope now, on this thread, and move a thread-free + * // SpanGuard into the job. + * SpanGuard span = std::move(scoped); + * jobQueue.addJob([g = std::move(span)]() mutable { doWork(); }); + * @endcode + * + * @note Thread safety: A ScopedSpanGuard must be constructed AND + * destroyed while the same LocalValue context store is active — its + * Scope binds to that store's context stack. This means the same + * thread, or the same JobQueue coroutine even if it resumes on another + * worker (coroutine-aware storage keeps the same store across the + * resume). `operator SpanGuard() &&` and the destructor must both run + * under the constructing store; both check this with an XRPL_ASSERT in + * debug/test/fuzzing builds. `operator SpanGuard() &&` pops the scope + * eagerly, so the resulting SpanGuard is thread-free. + * + * @note Known limitations: + * - Non-movable: cannot be stored in a container that relocates, and + * cannot be returned then move-assigned into an existing variable. + * Store the thread-free SpanGuard (via `operator SpanGuard() &&`) + * instead when a movable handle is needed. + * - Same attribute-removal limitation as SpanGuard. + */ +class ScopedSpanGuard +{ + struct ScopedImpl; + std::unique_ptr impl_; + + explicit ScopedSpanGuard(SpanGuard&& guard) noexcept; + +public: + /** + * Create a scoped span guarded by a TraceCategory flag and activate + * it on this thread. Equivalent to SpanGuard::span() wrapped in an + * active Scope. The span name is built as "prefix.name". + * @param cat Trace subsystem category. + * @param prefix Span name prefix (e.g. "rpc.command"). + * @param name Span name suffix (e.g. "submit"). + */ + ScopedSpanGuard(TraceCategory cat, std::string_view prefix, std::string_view name) noexcept; + + ~ScopedSpanGuard(); + + ScopedSpanGuard(ScopedSpanGuard&&) = delete; + ScopedSpanGuard& + operator=(ScopedSpanGuard&&) = delete; + ScopedSpanGuard(ScopedSpanGuard const&) = delete; + ScopedSpanGuard& + operator=(ScopedSpanGuard const&) = delete; + + // --- Static factory methods ---------------------------------------- + + /** + * Create a scoped span that always starts a fresh trace root and + * activate it on this thread. See SpanGuard::freshRoot(). + * @param cat Trace subsystem category. + * @param prefix Span name prefix. + * @param name Span name suffix. + * @return An active scoped root-span guard, or a null one if disabled. + */ + [[nodiscard]] static ScopedSpanGuard + freshRoot(TraceCategory cat, std::string_view prefix, std::string_view name) noexcept; + + // --- Child / linked span creation ---------------------------------- + + /** + * Create a scoped child span parented to this guard's active span + * and activate it on this thread. + * @param name Span name for the child. + * @return A new scoped guard, or a null one if this guard is inactive. + */ + [[nodiscard]] ScopedSpanGuard + childSpan(std::string_view name) const noexcept; + + /** + * Create a scoped child span parented to an explicit captured + * context and activate it on this thread. + * @param name Span name for the child. + * @param parentCtx Context captured via spanContext(). + * @return A new scoped guard, or a null one if parentCtx is invalid. + */ + [[nodiscard]] static ScopedSpanGuard + childSpan(std::string_view name, SpanContext const& parentCtx) noexcept; + + /** + * Create a scoped span linked (follows-from) to this guard's span + * and activate it on this thread. + * @param name Span name for the linked span. + * @return A new scoped guard, or a null one if this guard is inactive. + */ + [[nodiscard]] ScopedSpanGuard + linkedSpan(std::string_view name) const noexcept; + + /** + * Create a scoped span linked to an explicit captured context and + * activate it on this thread. + * @param name Span name for the linked span. + * @param linkCtx Context to link from. + * @return A new scoped guard, or a null one if linkCtx is invalid. + */ + [[nodiscard]] static ScopedSpanGuard + linkedSpan(std::string_view name, SpanContext const& linkCtx) noexcept; + + // --- Handoff bridge ------------------------------------------------- + + /** + * Pop the active Scope under the constructing context store and yield + * the span as a thread-free SpanGuard. Use to move a span into a job or + * onto another thread. Must be called while the constructing context + * store is active. + * @return The unscoped SpanGuard that now owns the span. + */ + operator SpanGuard() && noexcept; + + // --- Context capture ----------------------------------------------- + + /** + * Return a SpanContext referring to the owned guard's own span, for + * use as an explicit parent in childSpan(name, ctx) across threads. + * Forwards to the owned SpanGuard's spanContext(). Returns an invalid + * context if this guard is null. + * @return A SpanContext holding the owned guard's own span. + * @note This is NOT the thread's ambient context — use + * SpanGuard::threadLocalContext() for that. + */ + [[nodiscard]] SpanContext + spanContext() const noexcept; + + // --- Forwarding methods (delegate to the owned SpanGuard) ---------- + + /** + * Set a string attribute. No-op on a null guard. + */ + void + setAttribute(std::string_view key, std::string_view value) noexcept; + + /** + * Set a string attribute (C-string overload). No-op on a null guard. + */ + void + setAttribute(std::string_view key, char const* value) noexcept; + + /** + * Set an integer attribute. No-op on a null guard. + */ + void + setAttribute(std::string_view key, std::int64_t value) noexcept; + + /** + * Set a floating-point attribute. No-op on a null guard. + */ + void + setAttribute(std::string_view key, double value) noexcept; + + /** + * Set a boolean attribute. No-op on a null guard. + */ + void + setAttribute(std::string_view key, bool value) noexcept; + + /** + * Mark the span status as OK. No-op on a null guard. + */ + void + setOk() noexcept; + + /** + * Mark the span status as error. No-op on a null guard. + * @param description Optional human-readable error description. + */ + void + setError(std::string_view description = "") noexcept; + + /** + * Add a named event to the span's timeline. No-op on a null guard. + * @param name Event name. + */ + void + addEvent(std::string_view name) noexcept; + + /** + * Record an exception as a span event and mark status as error. + * No-op on a null guard. + * @param e The exception to record. + */ + void + recordException(std::exception const& e) noexcept; + + /** + * Mark this span for discard and end it immediately. discard() pops the + * scope right away (under the constructing context store) and discards + * the span. After discard() the guard is inert and its destructor is a + * no-op. + */ + void + discard() noexcept; + + /** + * @return true if this guard holds an active span. + */ + explicit + operator bool() const noexcept; +}; + +/** + * RAII activation of an already-owned span as the current context, WITHOUT + * taking ownership. Use to give a thread-free SpanGuard (e.g. one handed into + * a JobQueue job) ambient status for the duration of a synchronous, non-yielding + * worker body, so log lines emitted there carry the span's trace_id. The span + * is neither owned nor ended here — its owning SpanGuard still controls its + * lifetime. Non-copyable, non-movable; must be created and destroyed while the + * same context store is active (see ScopedSpanGuard). + * + * Dependency diagram: + * + * +---------------------------------------------+ + * | ScopedActivation | + * | (non-owning RAII span activation) | + * +---------------------------------------------+ + * | - impl_ : unique_ptr | + * | Impl { scope : otel::Scope; owner } | + * +---------------------------------------------+ + * | + (ctor) pushes span onto current store | + * | + (dtor) pops; does NOT end the span | + * +---------------------------------------------+ + * | activates (borrows, no ownership) + * +-----------+ ends the span + * | SpanGuard |----------------------------> (owner) + * +-----------+ + * + * @note Thread-safety: like ScopedSpanGuard, it pushes/pops the current + * LocalValue context store; construct and destroy it while the same store is + * active (asserted in debug). Must NOT outlive the SpanGuard whose span it + * activates. Intended as a short-lived stack local inside a worker body that + * runs to completion on one thread (no coro yield inside the activation). + * + * Example 1 — correlate a handed-off span's worker logs (primary use): + * @code + * // span is std::shared_ptr handed into this job body: + * auto activation = (span && *span) ? span->activate() : ScopedActivation{}; + * JLOG(j.info()) << "applied"; // carries span's trace_id + * // span is still owned/ended elsewhere; activation only pops here. + * @endcode + * + * Example 2 — edge case: a null guard yields a no-op activation: + * @code + * SpanGuard g; // null (telemetry off, or disabled category) + * auto a = g.activate(); // no-op; GetCurrent() unchanged + * @endcode + */ +class ScopedActivation +{ + struct Impl; + std::unique_ptr impl_; + friend class SpanGuard; + explicit ScopedActivation(std::unique_ptr impl) noexcept; + +public: + /** + * Construct a null / no-op activation. Its destructor does nothing and + * leaves the current context unchanged. Returned by SpanGuard::activate() + * for a null guard. + */ + ScopedActivation(); + + /** + * Pop the activated span off the current context store, restoring the + * prior context. Does NOT end the span (the owning SpanGuard does that). + * A null activation is a no-op. + */ + ~ScopedActivation(); + + ScopedActivation(ScopedActivation&&) = delete; + ScopedActivation& + operator=(ScopedActivation&&) = delete; + ScopedActivation(ScopedActivation const&) = delete; + ScopedActivation& + operator=(ScopedActivation const&) = delete; +}; + +/** + * Activate a span guard as the ambient context if it is live, else return a + * no-op activation. Accepts any handle (shared_ptr / optional) that is + * contextually convertible to bool and dereferences to a SpanGuard. Non-owning: + * the handle still owns/ends the span. Returns a null ScopedActivation when the + * handle is empty or the span is inactive (e.g. telemetry disabled). + * @param guard A shared_ptr or optional. + * @return An RAII activation; drop it to restore the prior context. + */ +template +[[nodiscard]] ScopedActivation +activateIfLive(SpanGuardHandle const& guard) +{ + if (guard && *guard) + return guard->activate(); + return ScopedActivation{}; +} + +// --------------------------------------------------------------------------- +// No-op stub (all inline, zero overhead, no OTel dependency) +// --------------------------------------------------------------------------- +#else // XRPL_ENABLE_TELEMETRY not defined + +/** + * No-op activation used when telemetry is disabled at compile time. Holds no + * state and does nothing on construction or destruction. Declared before + * SpanGuard so its inline activate() can return one by value. Non-copyable and + * non-movable to match the real ScopedActivation. + */ +class ScopedActivation +{ +public: + ScopedActivation() = default; + ~ScopedActivation() = default; + ScopedActivation(ScopedActivation&&) = delete; + ScopedActivation& + operator=(ScopedActivation&&) = delete; + ScopedActivation(ScopedActivation const&) = delete; + ScopedActivation& + operator=(ScopedActivation const&) = delete; +}; + +class SpanGuard +{ +public: + SpanGuard() = default; + ~SpanGuard() = default; + SpanGuard(SpanGuard&&) noexcept = default; + SpanGuard& + operator=(SpanGuard&&) noexcept = default; + SpanGuard(SpanGuard const&) = delete; + SpanGuard& + operator=(SpanGuard const&) = delete; + + [[nodiscard]] static SpanGuard + span(TraceCategory, std::string_view, std::string_view) noexcept + { + return {}; + } + + [[nodiscard]] static SpanGuard + freshRoot(TraceCategory, std::string_view, std::string_view) noexcept + { + return {}; + } + + // NOLINTBEGIN(readability-convert-member-functions-to-static) + [[nodiscard]] SpanGuard + childSpan(std::string_view) const noexcept + { + return {}; + } + [[nodiscard]] static SpanGuard + childSpan(std::string_view, SpanContext const&) noexcept + { + return {}; + } + [[nodiscard]] SpanGuard + linkedSpan(std::string_view) const noexcept + { + return {}; + } + [[nodiscard]] static SpanGuard + linkedSpan(std::string_view, SpanContext const&) noexcept + { + return {}; + } + + [[nodiscard]] static SpanGuard + hashSpan( + TraceCategory, + std::string_view, + std::uint8_t const*, + std::size_t, + SpanContext const* = nullptr) + { + return {}; + } + [[nodiscard]] static SpanGuard + hashSpan( + TraceCategory, + std::string_view, + std::uint8_t const*, + std::size_t, + std::uint8_t const*, + std::size_t, + std::uint8_t) + { + return {}; + } + + [[nodiscard]] SpanContext + spanContext() const noexcept + { + return {}; + } + [[nodiscard]] TraceBytes + getTraceBytes() const + { + return {}; + } + + static void + injectCurrentContextToProtobuf(protocol::TraceContext&) + { + } + // NOLINTEND(readability-convert-member-functions-to-static) + + [[nodiscard]] static SpanContext + threadLocalContext() noexcept + { + return {}; + } + + void + setAttribute(std::string_view, std::string_view) noexcept + { + } + void + setAttribute(std::string_view, char const*) noexcept + { + } + void + setAttribute(std::string_view, std::int64_t) noexcept + { + } + void + setAttribute(std::string_view, double) noexcept + { + } + void + setAttribute(std::string_view, bool) noexcept + { + } + + void + setOk() noexcept + { + } + void + setError(std::string_view = "") noexcept + { + } + void + addEvent(std::string_view) noexcept + { + } + void + addEvent(std::string_view, std::initializer_list) noexcept + { + } + void + recordException(std::exception const&) noexcept + { + } + void + discard() noexcept + { + } + + // NOLINTBEGIN(readability-convert-member-functions-to-static) + [[nodiscard]] ScopedActivation + activate() const noexcept + { + return {}; + } + // NOLINTEND(readability-convert-member-functions-to-static) + + explicit + operator bool() const noexcept + { + return false; + } +}; + +class ScopedSpanGuard +{ + // Private default ctor lets the inline factories/child methods build + // a no-op instance without exposing a public default constructor + // (mirrors the real class, which has no public default ctor). + ScopedSpanGuard() = default; + +public: + ScopedSpanGuard(TraceCategory, std::string_view, std::string_view) noexcept + { + } + ~ScopedSpanGuard() = default; + + ScopedSpanGuard(ScopedSpanGuard&&) = delete; + ScopedSpanGuard& + operator=(ScopedSpanGuard&&) = delete; + ScopedSpanGuard(ScopedSpanGuard const&) = delete; + ScopedSpanGuard& + operator=(ScopedSpanGuard const&) = delete; + + [[nodiscard]] static ScopedSpanGuard + freshRoot(TraceCategory, std::string_view, std::string_view) noexcept + { + return {}; + } + + // NOLINTBEGIN(readability-convert-member-functions-to-static) + [[nodiscard]] ScopedSpanGuard + childSpan(std::string_view) const noexcept + { + return {}; + } + [[nodiscard]] static ScopedSpanGuard + childSpan(std::string_view, SpanContext const&) noexcept + { + return {}; + } + [[nodiscard]] ScopedSpanGuard + linkedSpan(std::string_view) const noexcept + { + return {}; + } + [[nodiscard]] static ScopedSpanGuard + linkedSpan(std::string_view, SpanContext const&) noexcept + { + return {}; + } + + [[nodiscard]] SpanContext + spanContext() const noexcept + { + return {}; + } + // NOLINTEND(readability-convert-member-functions-to-static) + + operator SpanGuard() && noexcept + { + return {}; + } + + void + setAttribute(std::string_view, std::string_view) noexcept + { + } + void + setAttribute(std::string_view, char const*) noexcept + { + } + void + setAttribute(std::string_view, std::int64_t) noexcept + { + } + void + setAttribute(std::string_view, double) noexcept + { + } + void + setAttribute(std::string_view, bool) noexcept + { + } + + void + setOk() noexcept + { + } + void + setError(std::string_view = "") noexcept + { + } + void + addEvent(std::string_view) noexcept + { + } + void + recordException(std::exception const&) noexcept + { + } + void + discard() noexcept + { + } + + explicit + operator bool() const noexcept + { + return false; + } +}; + +/** + * No-op counterpart to the telemetry-enabled activateIfLive(). Same signature + * and handle contract (shared_ptr / optional of SpanGuard); always yields a + * null ScopedActivation because a stub guard is never live. + * @param guard A shared_ptr or optional. + * @return A null (no-op) ScopedActivation. + */ +template +[[nodiscard]] ScopedActivation +activateIfLive(SpanGuardHandle const& guard) +{ + if (guard && *guard) + return guard->activate(); + return ScopedActivation{}; +} + +#endif // XRPL_ENABLE_TELEMETRY + +} // namespace xrpl::telemetry diff --git a/include/xrpl/telemetry/SpanNames.h b/include/xrpl/telemetry/SpanNames.h new file mode 100644 index 00000000000..b848f96c039 --- /dev/null +++ b/include/xrpl/telemetry/SpanNames.h @@ -0,0 +1,171 @@ +#pragma once + +/** + * Compile-time string concatenation utility and shared telemetry constants. + * + * Provides StaticStr — a compile-time string buffer that implicitly + * converts to std::string_view — and join() for dot-separated concatenation. + * Module-specific span names (e.g. RPC, consensus) live in their respective + * modules and build upon these shared primitives. + * + * @note These constants are NOT guarded by XRPL_ENABLE_TELEMETRY because + * call sites reference them even when SpanGuard methods are no-ops + * (the no-op stubs still accept string_view parameters). The compiler + * elides all inline constexpr values whose only uses are in dead code. + * + * @note Json::StaticString (jss.h) is a pointer wrapper without + * concatenation support. boost::static_string is not constexpr. + * StaticStr exists specifically for compile-time dot-join composition. + * + * Naming conventions (see spec 2026-05-13-span-attr-naming-design): + * - Per-span attribute keys: bare field name (span name carries the domain). + * - Collision qualifier: _ when bare name collides across + * domains or with OTel reserved `status` (e.g. rpc_status, grpc_status). + * - Shared cross-span attributes: _ (underscore) form + * (e.g. tx_hash, peer_id, ledger_seq, consensus_round). + * - Resource attribute keys: xrpl.. (dotted) form is + * RESERVED for process-identity attributes set once at startup on the + * OTel resource (e.g. xrpl.network.id, xrpl.network.type). Do not use + * this form for span attributes — it parses awkwardly in TraceQL and + * blurs the resource/span scope distinction. + * - Span prefixes: [.]. + */ + +#include +#include + +namespace xrpl::telemetry { + +// ===== Compile-time string utility ========================================= + +/** + * Fixed-size character buffer for compile-time string operations. + * Implicitly converts to std::string_view at zero cost. + */ +template +struct StaticStr +{ + char data[N + 1]{}; + static constexpr std::size_t size = N; + + constexpr StaticStr() = default; + + constexpr explicit StaticStr(char const (&str)[N + 1]) + { + for (std::size_t i = 0; i <= N; ++i) + data[i] = str[i]; + } + + constexpr + operator std::string_view() const noexcept + { + return {data, N}; + } +}; + +/** + * Deduction guide: StaticStr from string literal. + */ +template +StaticStr(char const (&)[N]) -> StaticStr; + +/** + * Create a StaticStr from a string literal. + */ +template +constexpr auto +makeStr(char const (&str)[N]) +{ + return StaticStr(str); +} + +/** + * Concatenate two StaticStr values with a dot separator. + */ +template +constexpr auto +join(StaticStr const& lhs, StaticStr const& rhs) +{ + constexpr std::size_t len = A + 1 + B; // lhs + '.' + rhs + StaticStr result; + std::size_t pos = 0; + for (std::size_t i = 0; i < A; ++i) + result.data[pos++] = lhs.data[i]; + result.data[pos++] = '.'; + for (std::size_t i = 0; i < B; ++i) + result.data[pos++] = rhs.data[i]; + result.data[pos] = '\0'; + return result; +} + +// ===== Shared root segments ================================================ + +namespace seg { +inline constexpr auto xrpl = makeStr("xrpl"); +inline constexpr auto rpc = makeStr("rpc"); +inline constexpr auto tx = makeStr("tx"); +inline constexpr auto consensus = makeStr("consensus"); +inline constexpr auto peer = makeStr("peer"); +inline constexpr auto ledger = makeStr("ledger"); +inline constexpr auto network = makeStr("network"); +inline constexpr auto link = makeStr("link"); +} // namespace seg + +// ===== Shared attribute keys (used across modules) ========================= + +namespace attr { +inline constexpr auto networkId = join(join(seg::xrpl, seg::network), makeStr("id")); +inline constexpr auto networkType = join(join(seg::xrpl, seg::network), makeStr("type")); + +/** + * Canonical shared attrs (rule 5 — _ underscore form). + * + * Per the naming convention header note: shared cross-span attribute + * keys use the underscore form, reserving the dotted xrpl.. + * form for resource attributes set on the OTel resource at startup. + * Defined once here, aliased by domain-specific headers. These are + * literal underscore-joined names, not dot-joined via `join()`, since + * `join()` always inserts `.` between its arguments. + */ +inline constexpr auto txHash = makeStr("tx_hash"); +inline constexpr auto peerId = makeStr("peer_id"); +inline constexpr auto ledgerSeq = makeStr("ledger_seq"); + +/** + * Shared close-time attrs — bare names, reused by consensus and ledger. + */ +inline constexpr auto closeTime = makeStr("close_time"); +inline constexpr auto closeTimeCorrect = makeStr("close_time_correct"); +inline constexpr auto closeResolutionMs = makeStr("close_resolution_ms"); +/** + * Shared validation attrs — reused by the consensus and peer validation + * spans. Same concept, same key on every span; the span name tells them + * apart, so neither is emitter-prefixed. `ledgerHash` is a ledger-object + * property (bare, like ledgerSeq); `fullValidation` is the is-full-validation + * flag. Never the dotted xrpl. form (reserved for resource attrs). + */ +inline constexpr auto ledgerHash = makeStr("ledger_hash"); +inline constexpr auto fullValidation = makeStr("full_validation"); + +/** + * Shared "ledger being worked on" attrs — the open/tentative or in-flight + * consensus-build ledger a transaction is applied into, NOT an established or + * validated ledger (that is `ledgerSeq`, set on ledger.build / consensus.round). + * Named after the RPC field `ledger_current_index` and the `currentLedgerSeq` + * log usage. Reused by the tx lifecycle, apply-pipeline, and TxQ spans so a + * transaction's work can be correlated to the ledger it targeted. + * `currentLedgerHash` is the current view's parent-ledger hash, which equals the + * consensus.round deterministic trace-id seed on the consensus-build path. + */ +inline constexpr auto currentLedgerSeq = makeStr("current_ledger_seq"); +inline constexpr auto currentLedgerHash = makeStr("current_ledger_hash"); +} // namespace attr + +// ===== Shared attribute values ============================================= + +namespace attr_val { +inline constexpr auto success = makeStr("success"); +inline constexpr auto error = makeStr("error"); +} // namespace attr_val + +} // namespace xrpl::telemetry diff --git a/include/xrpl/telemetry/Telemetry.h b/include/xrpl/telemetry/Telemetry.h new file mode 100644 index 00000000000..b9ca1dca642 --- /dev/null +++ b/include/xrpl/telemetry/Telemetry.h @@ -0,0 +1,477 @@ +#pragma once + +/** + * Abstract interface for OpenTelemetry distributed tracing. + * + * Provides the Telemetry base class that all components use to create trace + * spans. Three concrete implementations exist, selected at construction time + * by makeTelemetry(): + * + * - TelemetryImpl (Telemetry.cpp): real OTel SDK integration, compiled + * only when XRPL_ENABLE_TELEMETRY is defined and enabled at runtime. + * - NullTelemetry (NullTelemetry.cpp): no-op stub used when telemetry is + * disabled at compile time or runtime. + * - NullTelemetryOtel (Telemetry.cpp): no-op stub that still depends on + * the OTel API (used during transition or for testing). + * + * Inheritance / dependency diagram: + * + * +--------------------+ + * | Telemetry | (abstract, this file) + * | <> | + * +---------+----------+ + * | + * +---------+-----------+-------------------+ + * | | | + * +---+------------+ +-----+---------+ +------+----------+ + * | TelemetryImpl | | NullTelemetry | | NullTelemetryOtel| + * | (Telemetry.cpp)| |(NullTelemetry | | (Telemetry.cpp) | + * | OTel SDK | | .cpp) | | noop w/ OTel API | + * +----------------+ +---------------+ +------------------+ + * + * The Setup struct holds all configuration parsed from the [telemetry] + * section of xrpld.cfg. See TelemetryConfig.cpp for the parser and + * cfg/xrpld-example.cfg for the available options. + * + * OTel SDK headers are conditionally included behind XRPL_ENABLE_TELEMETRY + * so that builds without telemetry have zero dependency on opentelemetry-cpp. + * + * Usage examples: + * + * 1. Root span at a subsystem entry point (typical usage): + * @code + * #include + * using namespace xrpl::telemetry; + * + * // In an RPC handler dispatch: + * auto guard = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::command, commandName); + * guard.setAttribute(rpc_span::attr::command, commandName); + * // ... process request + * // guard destructor automatically ends the span on scope exit + * @endcode + * + * 2. Child span for a sub-operation (scoped child): + * @code + * auto parent = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::process); + * { + * auto child = parent.childSpan(rpc_span::op::process); + * child.setAttribute(rpc_span::attr::version, apiVersion); + * // child ends here + * } + * @endcode + * + * 3. Unrelated span (cross-scope, same thread): + * @code + * // gRPC and RPC handlers can be active simultaneously + * auto grpcSpan = SpanGuard::span( + * TraceCategory::Rpc, grpc_span::prefix::grpc, grpc_span::attr::method); + * auto rpcSpan = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::command, commandName); + * // both spans end on scope exit + * @endcode + * + * 4. Cross-thread context propagation: + * @code + * // Thread A: take a handle to the parent span's own context + * auto ctx = parentGuard.spanContext(); + * + * // Thread B: create child span with explicit parent + * auto child = SpanGuard::childSpan(rpc_span::op::process, ctx); + * @endcode + * + * @note Thread safety: The Telemetry interface is safe for concurrent reads + * (isEnabled, shouldTrace*, getTracer, startSpan) after start() completes. + * setServiceInstanceId() must be called before start() and is not thread-safe. + * The OTel SDK's TracerProvider and Tracer are internally thread-safe. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifdef XRPL_ENABLE_TELEMETRY +#include +#include +#include +#include +#include +#include +#endif + +namespace xrpl::telemetry { + +#ifdef XRPL_ENABLE_TELEMETRY +/** + * OTel instrumentation scope (tracer) name. Identifies this library as the + * source of spans; distinct from the `service.name` resource attribute + * (Setup::serviceName), which is config-overridable. + */ +inline constexpr std::string_view kTracerName{"xrpld"}; + +/** + * OTel instrumentation scope (meter) name. Identifies this library as the + * source of metrics; symmetric with kTracerName for the tracing side. + */ +inline constexpr std::string_view kMeterName{"xrpld"}; + +/** + * OTel instrumentation scope version reported for the meter. + */ +inline constexpr std::string_view kMeterVersion{"1.0.0"}; +#endif + +class Telemetry +{ + /** + * Global singleton pointer, set by start()/stop() in the active + * implementation. Allows SpanGuard factory methods to access the + * Telemetry instance without callers passing it explicitly. + * + * Atomic with acquire/release ordering: start()/stop() store on + * the initialization thread, factory methods load on worker threads. + * @see setInstance(), getInstance() + */ + inline static std::atomic instance{nullptr}; + +public: + /** + * Get the global Telemetry instance. + * @return Pointer to the active instance, or nullptr if not started. + */ + static Telemetry* + getInstance() + { + return instance.load(std::memory_order_acquire); + } + + /** + * Set the global Telemetry instance. + * Called by start()/stop() in concrete implementations. + * Tests can call this with a mock to override the global instance. + * @param t Pointer to the Telemetry instance, or nullptr to clear. + */ + static void + setInstance(Telemetry* t) + { + instance.store(t, std::memory_order_release); + } + + /** + * Configuration parsed from the [telemetry] section of xrpld.cfg. + * + * All fields have sensible defaults so the section can be minimal + * or omitted entirely. See TelemetryConfig.cpp for the parser. + */ + struct Setup + { + /** + * Master switch: true to enable tracing at runtime. + */ + bool enabled = false; + + /** + * OTel resource attribute `service.name`. + */ + std::string serviceName = "xrpld"; + + /** + * OTel resource attribute `service.version` (set from BuildInfo). + */ + std::string serviceVersion; + + /** + * OTel resource attribute `service.instance.id` (defaults to node + * public key). + */ + std::string serviceInstanceId; + + /** + * OTLP/HTTP endpoint URL where spans are sent. + */ + std::string exporterEndpoint = "http://localhost:4318/v1/traces"; + + /** + * Whether to use TLS for the exporter connection. + */ + bool useTls = false; + + /** + * Path to a CA certificate bundle for TLS verification. + */ + std::string tlsCertPath; + + /** + * Path to this node's client certificate (PEM), presented to the + * collector for mutual TLS. Empty disables client-side auth, in + * which case only server (one-way) TLS is used. + */ + std::string tlsClientCertPath; + + /** + * Path to the private key (PEM) for tlsClientCertPath. Required + * whenever tlsClientCertPath is set. + */ + std::string tlsClientKeyPath; + + /** + * Head-based sampling ratio. Intentionally fixed at 1.0 (sample + * everything) and NOT read from config. A per-node ratio would let + * nodes make divergent keep/drop decisions for the same distributed + * trace, producing broken/partial traces. The ratio sampler is wrapped + * in a ParentBasedSampler (see Telemetry.cpp) so spans inheriting a + * remote parent honor the upstream sampled flag. Volume reduction is + * delegated to the collector's tail sampling; for node-local post-hoc + * dropping see SpanGuard::discard(). + */ + static constexpr double samplingRatio = 1.0; + + /** + * Maximum number of spans per batch export. + */ + std::uint32_t batchSize = 512; + + /** + * Delay between batch exports. + */ + std::chrono::milliseconds batchDelay = std::chrono::milliseconds{5000}; + + /** + * Maximum number of spans queued before dropping. + */ + std::uint32_t maxQueueSize = 2048; + + /** + * Network identifier, added as an OTel resource attribute. + */ + std::uint32_t networkId = 0; + + /** + * Network type label (e.g. "mainnet", "testnet", "devnet"). + */ + std::string networkType = "mainnet"; + + /** + * Enable tracing for transaction processing. + */ + bool traceTransactions = true; + + /** + * Enable tracing for consensus rounds. + */ + bool traceConsensus = true; + + /** + * Enable tracing for RPC request handling. + */ + bool traceRpc = true; + + /** + * Enable tracing for peer-to-peer messages (enabled by default; + * high volume). + */ + bool tracePeer = true; + + /** + * Enable tracing for ledger close/accept. + */ + bool traceLedger = true; + + /** + * Strategy for cross-node consensus trace correlation. + * "deterministic" — derive trace_id from ledger hash so all + * validators in the same round share the same trace_id. + * "attribute" — random trace_id, correlate via ledger_id attribute. + */ + std::string consensusTraceStrategy = "deterministic"; + }; + + virtual ~Telemetry() = default; + + /** + * Update the service instance ID (OTel resource attribute + * `service.instance.id`). + * + * Must be called before start(). The node public key is not available + * when Telemetry is constructed (during the ApplicationImp member + * initializer list), so this setter allows Application::setup() to + * inject the identity once nodeIdentity_ is known. + * + * @param id The node's base58-encoded public key or custom identifier. + */ + virtual void + setServiceInstanceId(std::string const& id) + { + // Default no-op for NullTelemetry implementations. + (void)id; + } + + /** + * Initialize the tracing pipeline (exporter, processor, provider). + * Call after construction. + */ + virtual void + start() = 0; + + /** + * Flush pending spans and shut down the tracing pipeline. + * Call before destruction. + */ + virtual void + stop() = 0; + + /** + * @return true if this instance is actively exporting spans. + */ + [[nodiscard]] virtual bool + isEnabled() const = 0; + + /** + * @return true if transaction processing should be traced. + */ + [[nodiscard]] virtual bool + shouldTraceTransactions() const = 0; + + /** + * @return true if consensus rounds should be traced. + */ + [[nodiscard]] virtual bool + shouldTraceConsensus() const = 0; + + /** + * @return true if RPC request handling should be traced. + */ + [[nodiscard]] virtual bool + shouldTraceRpc() const = 0; + + /** + * @return true if peer-to-peer messages should be traced. + */ + [[nodiscard]] virtual bool + shouldTracePeer() const = 0; + + /** + * @return true if ledger close/accept should be traced. + */ + [[nodiscard]] virtual bool + shouldTraceLedger() const = 0; + + /** + * @return The configured consensus trace correlation strategy. + */ + [[nodiscard]] virtual std::string const& + getConsensusTraceStrategy() const = 0; + +#ifdef XRPL_ENABLE_TELEMETRY + /** + * Get or create a named tracer instance. + * + * @param name Tracer name used to identify the instrumentation library. + * @return A shared pointer to the Tracer. + */ + virtual opentelemetry::nostd::shared_ptr + getTracer(std::string_view name = kTracerName) = 0; + + /** + * Get or create a named meter instance. + * + * Returns the raw OTel Meter, giving developers direct access to the + * full metrics API. From the returned Meter any instrument type can be + * created: Counter, UpDownCounter, Gauge, and Histogram (synchronous), + * plus their observable/async variants (ObservableCounter, + * ObservableUpDownCounter, ObservableGauge). + * + * @param name Meter name used to identify the instrumentation scope. + * @return A shared pointer to the Meter. + */ + virtual opentelemetry::nostd::shared_ptr + getMeter(std::string_view name = kMeterName) = 0; + + /** + * Start a new span on the current thread's context. + * + * The span becomes a child of the current active span (if any) via + * OpenTelemetry's context propagation. + * + * @param name Span name (typically "rpc.command."). + * @param kind The span kind (defaults to kInternal). Possible values: + * - kInternal: default, in-process operation + * - kServer: incoming synchronous request (e.g. RPC) + * - kClient: outgoing synchronous request + * - kProducer: async message send (e.g. peer broadcast) + * - kConsumer: async message receive + * @return A shared pointer to the new Span. + */ + virtual opentelemetry::nostd::shared_ptr + startSpan( + std::string_view name, + opentelemetry::trace::SpanKind kind = opentelemetry::trace::SpanKind::kInternal) = 0; + + /** + * Start a new span with an explicit parent context. + * + * Use this overload when the parent span is not on the current + * thread's context stack (e.g. cross-thread trace propagation). + * + * @param name Span name. + * @param parentContext The parent span's context. + * @param kind The span kind (defaults to kInternal). + * @return A shared pointer to the new Span. + */ + virtual opentelemetry::nostd::shared_ptr + startSpan( + std::string_view name, + opentelemetry::context::Context const& parentContext, + opentelemetry::trace::SpanKind kind = opentelemetry::trace::SpanKind::kInternal) = 0; +#endif +}; + +/** + * Create a Telemetry instance. + * + * Returns a TelemetryImpl when setup.enabled is true, or a + * NullTelemetry no-op stub otherwise. + * + * @param setup Configuration from the [telemetry] config section. + * @param journal Journal for log output during initialization. + */ +std::unique_ptr +makeTelemetry(Telemetry::Setup const& setup, beast::Journal journal); + +/** + * Parse the [telemetry] config section into a Setup struct. + * + * @param section The [telemetry] config section. + * @param nodePublicKey Node public key, used as default instance ID. + * @param version Build version string. + * @param networkId Network identifier from [network_id] config + * (0 = mainnet, 1 = testnet, 2 = devnet). + * @return A populated Setup struct with defaults for missing values. + */ +Telemetry::Setup +makeTelemetrySetup( + Section const& section, + std::string const& nodePublicKey, + std::string const& version, + std::uint32_t networkId); + +/** + * Derive a human-readable network type label from the numeric network ID. + * + * Shared by the trace and metric export paths so both stamp the same + * `xrpl.network.type` resource attribute value. + * + * @param networkId The network identifier from [network_id] config. + * @return "mainnet" (0), "testnet" (1), "devnet" (2), or "unknown". + */ +std::string +networkTypeFromId(std::uint32_t networkId); + +} // namespace xrpl::telemetry diff --git a/include/xrpl/telemetry/TraceContextPropagator.h b/include/xrpl/telemetry/TraceContextPropagator.h new file mode 100644 index 00000000000..e8d851e53e2 --- /dev/null +++ b/include/xrpl/telemetry/TraceContextPropagator.h @@ -0,0 +1,115 @@ +#pragma once + +/** + * Utilities for trace context propagation across nodes. + * + * Provides serialization/deserialization of OTel trace context to/from + * Protocol Buffer TraceContext messages (P2P cross-node propagation). + * Wired into the P2P message flow via PropagationHelpers.h for + * TMTransaction, TMProposeSet, and TMValidation messages. + * + * Only compiled when XRPL_ENABLE_TELEMETRY is defined. + * + * @see PropagationHelpers.h (high-level inject helpers), + * TxTracing.h (transaction receive-side extraction), + * ConsensusReceiveTracing.h (proposal/validation receive-side). + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace xrpl::telemetry { + +/** + * Extract OTel context from a protobuf TraceContext message. + * + * @param proto The protobuf TraceContext received from a peer. + * @return An OTel Context with the extracted parent span, or an empty + * context if the protobuf fields are missing or invalid. + */ +inline opentelemetry::context::Context +extractFromProtobuf(protocol::TraceContext const& proto) +{ + namespace trace = opentelemetry::trace; + + // Reject malformed or all-zero ids from the peer before trusting + // them as a parent. See TraceContextValidation.h. + if (!isValidTraceContext(proto)) + { + return opentelemetry::context::Context{}; + } + + auto const* rawTraceId = reinterpret_cast(proto.trace_id().data()); + auto const* rawSpanId = reinterpret_cast(proto.span_id().data()); + trace::TraceId const traceId( + opentelemetry::nostd::span(rawTraceId, 16)); + trace::SpanId const spanId(opentelemetry::nostd::span(rawSpanId, 8)); + trace::TraceFlags const flags( + proto.has_trace_flags() ? static_cast(proto.trace_flags()) + : static_cast(0)); + + trace::SpanContext const spanCtx(traceId, spanId, flags, /* remote = */ true); + + return opentelemetry::context::Context{}.SetValue( + trace::kSpanKey, + opentelemetry::nostd::shared_ptr(new trace::DefaultSpan(spanCtx))); +} + +/** + * Inject the current span's trace context into a protobuf TraceContext. + * + * @param ctx The OTel context containing the span to propagate. + * @param proto The protobuf TraceContext to populate. + */ +inline void +injectToProtobuf(opentelemetry::context::Context const& ctx, protocol::TraceContext& proto) +{ + namespace trace = opentelemetry::trace; + + auto const span = trace::GetSpan(ctx); + if (!span) + return; + + auto const& spanCtx = span->GetContext(); + if (!spanCtx.IsValid()) + return; + + // Serialize trace_id (16 bytes) + auto const& traceId = spanCtx.trace_id(); + proto.set_trace_id(traceId.Id().data(), trace::TraceId::kSize); + + // Serialize span_id (8 bytes) + auto const& spanId = spanCtx.span_id(); + proto.set_span_id(spanId.Id().data(), trace::SpanId::kSize); + + // Serialize flags + proto.set_trace_flags(spanCtx.trace_flags().flags()); + + // TODO(observability/secure-OTel): the protobuf TraceContext message + // also carries `trace_state` (field 4), which is currently neither + // populated here nor read by extractFromProtobuf above. The field is + // reserved for the secure tracing pipeline outlined in + // OpenTelemetryPlan/secure-OTel.md, where an authenticated token in + // tracestate will let receivers reject spoofed/poisoned trace context. + // Wire trace_state through inject/extract once the consumer lands. +} + +} // namespace xrpl::telemetry + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/include/xrpl/telemetry/TraceContextValidation.h b/include/xrpl/telemetry/TraceContextValidation.h new file mode 100644 index 00000000000..e69b2d17ca1 --- /dev/null +++ b/include/xrpl/telemetry/TraceContextValidation.h @@ -0,0 +1,96 @@ +#pragma once + +/** + * Validation predicates for peer-supplied trace context. + * + * A protobuf TraceContext arrives inside untrusted peer messages + * (TMTransaction, TMProposeSet, TMValidation). Before a receiving node + * builds a span from it, the context must be checked: a peer can send a + * malformed or all-zero trace_id/span_id, either by accident or to + * pollute traces. This header is the single place those checks live, so + * every receive site agrees on what "valid" means. + * + * Validity follows the OpenTelemetry spec: a trace_id is 16 bytes, a + * span_id is 8 bytes, and an all-zero id is invalid and must not be + * used as a parent. + * + * Dependency diagram: + * + * protocol::TraceContext (untrusted, from peer) + * | + * v + * isValidTraceContext() / isValidSpanId() / isValidTraceId() + * | + * +--- valid ----> build child span from peer context + * | + * +--- invalid --> ignore peer context, start fresh span + * + * This header depends only on the generated protobuf type. It pulls in + * no OpenTelemetry headers, so receive sites that use SpanGuard keep + * SpanGuard's encapsulation of OTel types. + * + * @note Thread-safe: all functions are pure and operate only on their + * arguments. No shared state. + * + * Usage: + * @code + * // Full parent context (both ids come from the peer): + * if (isValidTraceContext(msg.trace_context())) + * buildChildSpan(...); + * + * // Span-only (trace_id is derived locally, not from the peer): + * if (tc.has_span_id() && isValidSpanId(tc.span_id())) + * buildChildSpan(...); + * @endcode + */ + +#include + +#include +#include + +namespace xrpl::telemetry { + +/** + * True if the bytes are a valid OTel trace_id: 16 bytes, not all-zero. + * + * @param traceId The raw trace_id bytes from a protobuf TraceContext. + * @return true if usable as a trace identifier, false otherwise. + */ +inline bool +isValidTraceId(std::string const& traceId) +{ + return traceId.size() == 16 && std::ranges::any_of(traceId, [](char c) { return c != 0; }); +} + +/** + * True if the bytes are a valid OTel span_id: 8 bytes, not all-zero. + * + * @param spanId The raw span_id bytes from a protobuf TraceContext. + * @return true if usable as a span identifier, false otherwise. + */ +inline bool +isValidSpanId(std::string const& spanId) +{ + return spanId.size() == 8 && std::ranges::any_of(spanId, [](char c) { return c != 0; }); +} + +/** + * True if the context carries a usable parent: a valid trace_id and a + * valid span_id together. + * + * Use this where both ids are taken from the peer (consensus receive, + * generic extraction). The transaction path derives its trace_id + * locally from the txID, so it checks isValidSpanId() alone instead. + * + * @param tc The protobuf TraceContext received from a peer. + * @return true if both ids are present and valid, false otherwise. + */ +inline bool +isValidTraceContext(protocol::TraceContext const& tc) +{ + return tc.has_trace_id() && isValidTraceId(tc.trace_id()) && tc.has_span_id() && + isValidSpanId(tc.span_id()); +} + +} // namespace xrpl::telemetry diff --git a/include/xrpl/tx/detail/TxApplySpanNames.h b/include/xrpl/tx/detail/TxApplySpanNames.h new file mode 100644 index 00000000000..b44019fc628 --- /dev/null +++ b/include/xrpl/tx/detail/TxApplySpanNames.h @@ -0,0 +1,150 @@ +#pragma once + +/** + * Compile-time span name constants for the transaction apply pipeline. + * + * Defines the span names and attribute keys used by the three apply-pipeline + * stages — preflight, preclaim, and transactor (apply) — that run inside the + * library (`src/libxrpl/tx/`). Built on the StaticStr/join() primitives from + * . + * + * Why a separate header from TxSpanNames.h: + * TxSpanNames.h lives under src/xrpld/ (daemon) and serves the overlay/app + * lifecycle spans (tx.receive, tx.process). Library code (applySteps.cpp, + * Transactor.cpp) must not depend on daemon headers, so the apply-pipeline + * constants live here instead. The attribute strings ("tx_type", + * "ter_result", "applied") intentionally match TxSpanNames.h so the collector + * spanmetrics connector aggregates both sets under the same dimensions. + * + * Span hierarchy (deterministic trace_id derived from txID[0:16]): + * + * The three stages run sequentially and often on different threads, so they + * do not auto-parent. Each uses a hash-derived trace_id keyed on the same + * transaction id, placing all three under one trace without context + * propagation. A transaction that hard-fails preflight or preclaim never + * reaches the transactor span — the stage attribute identifies where it + * stopped. + * + * +-----------------------------------------------------------+ + * | trace_id = txID[0:16] | + * | | + * | +-------------------+ +------------------+ +-------+ | + * | | tx.preflight | | tx.preclaim | | tx. | | + * | | stage=preflight |-->| stage=preclaim |-->| trans | | + * | | tx_type | | tx_type | | actor | | + * | | ter_result | | ter_result | | stage=| | + * | +-------------------+ +------------------+ | apply | | + * | stateless checks ledger-aware checks +-------+ | + * | (signature, fields) (sequence, fee) applies | + * +-----------------------------------------------------------+ + * + * Usage: + * @code + * #include + * using namespace telemetry; + * + * // preflight() / preclaim() use hashSpan with a full span name: + * auto span = SpanGuard::hashSpan( + * TraceCategory::Transactions, tx_apply_span::preflight, + * txID.data(), txID.kBytes); + * span.setAttribute(tx_apply_span::attr::stage, tx_apply_span::val::preflight); + * span.setAttribute(tx_apply_span::attr::terResult, transToken(ter).c_str()); + * @endcode + * + * @code + * // Transactor::operator() also uses hashSpan on the same txID so it + * // co-traces with preflight and preclaim under one trace_id: + * auto span = SpanGuard::hashSpan( + * TraceCategory::Transactions, tx_apply_span::transactor, + * txID.data(), txID.kBytes); + * span.setAttribute(tx_apply_span::attr::stage, tx_apply_span::val::apply); + * @endcode + */ + +#include + +namespace xrpl::telemetry::tx_apply_span { + +// ===== Span operation suffixes ============================================= + +namespace op { +/** + * "preflight" — stateless transaction checks (suffix form). + */ +inline constexpr auto preflight = makeStr("preflight"); +/** + * "preclaim" — ledger-aware checks before fee claim (suffix form). + */ +inline constexpr auto preclaim = makeStr("preclaim"); +/** + * "transactor" — the apply stage (suffix form, used with span()). + */ +inline constexpr auto transactor = makeStr("transactor"); +} // namespace op + +// ===== Full span names (tx.) =========================================== + +/** + * "tx.preflight" — full name for hashSpan() at the preflight stage. + */ +inline constexpr auto preflight = join(seg::tx, op::preflight); +/** + * "tx.preclaim" — full name for hashSpan() at the preclaim stage. + */ +inline constexpr auto preclaim = join(seg::tx, op::preclaim); +/** + * "tx.transactor" — full name for hashSpan() at the apply stage. Shares the + * txID-derived trace_id so it co-traces with tx.preflight and tx.preclaim. + */ +inline constexpr auto transactor = join(seg::tx, op::transactor); + +// ===== Attribute keys ====================================================== + +namespace attr { +/** + * Shared "ledger being worked on" attrs (defined in SpanNames.h). Set on + * tx.preclaim and tx.transactor (both run against a view whose seq() is the + * ledger being applied into). tx.preflight is stateless (no view) and is the + * documented exception — it carries neither. + */ +using ::xrpl::telemetry::attr::currentLedgerHash; +using ::xrpl::telemetry::attr::currentLedgerSeq; + +/** + * "stage" — which apply-pipeline stage this span represents. Drives the + * collector spanmetrics `stage` dimension for per-stage RED metrics. + */ +inline constexpr auto stage = makeStr("stage"); +/** + * "tx_type" — transaction type name (e.g., "Payment", "OfferCreate"). + * Matches tx_span::attr::txType so both share the spanmetrics dimension. + */ +inline constexpr auto txType = makeStr("tx_type"); +/** + * "ter_result" — engine result code after the stage (e.g., "tesSUCCESS"). + */ +inline constexpr auto terResult = makeStr("ter_result"); +/** + * "applied" — whether the transaction was applied to the ledger (apply only). + */ +inline constexpr auto applied = makeStr("applied"); +} // namespace attr + +// ===== Attribute values (stage names) ====================================== + +namespace val { +/** + * "preflight" — value of the stage attribute on tx.preflight. + */ +inline constexpr auto preflight = makeStr("preflight"); +/** + * "preclaim" — value of the stage attribute on tx.preclaim. + */ +inline constexpr auto preclaim = makeStr("preclaim"); +/** + * "apply" — value of the stage attribute on tx.transactor. + */ +inline constexpr auto apply = makeStr("apply"); +} // namespace val + +} // namespace xrpl::telemetry::tx_apply_span diff --git a/sanitizers/suppressions/ubsan.supp b/sanitizers/suppressions/ubsan.supp index 7e3e02f8553..de2a66e64ae 100644 --- a/sanitizers/suppressions/ubsan.supp +++ b/sanitizers/suppressions/ubsan.supp @@ -102,6 +102,10 @@ undefined:nudb # Snappy compression library intentional overflows unsigned-integer-overflow:snappy.cc +# OpenTelemetry SDK intentional overflows in hash functions +unsigned-integer-overflow:opentelemetry/sdk/common/attributemap_hash.h +unsigned-integer-overflow:opentelemetry + # Abseil intentional overflows in hashing, RNG and time arithmetic. # Matched at library scope (like boost above): the wraparound is by design # across many absl files (hash mixing, raw_hash_set probing, duration math, diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index d1e54a515fe..46a3829e78d 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -7,7 +7,17 @@ #include #include +#ifdef XRPL_ENABLE_TELEMETRY +#include +#include +#include +#include +#include +#include +#endif // XRPL_ENABLE_TELEMETRY + #include +#include #include #include #include @@ -291,6 +301,42 @@ Logs::format( break; } +#ifdef XRPL_ENABLE_TELEMETRY + // Inject OTel trace context when an active span exists on this thread. + // Checks the thread-local context value directly to avoid the heap + // allocation that GetSpan() performs on the no-span path. + { + auto context = opentelemetry::context::RuntimeContext::GetCurrent(); + auto spanValue = context.GetValue(opentelemetry::trace::kSpanKey); + if (opentelemetry::nostd::holds_alternative< + opentelemetry::nostd::shared_ptr>(spanValue)) + { + auto span = opentelemetry::nostd::get< + opentelemetry::nostd::shared_ptr>(spanValue); + auto spanCtx = span->GetContext(); + if (spanCtx.IsValid()) + { + // Hex widths of a W3C trace context: 16-byte trace_id and + // 8-byte span_id render to 32 and 16 lowercase hex chars. + constexpr std::size_t kTraceIdHexLen = 32; + constexpr std::size_t kSpanIdHexLen = 16; + constexpr auto kTraceIdPrefix = "trace_id="; + constexpr auto kSpanIdPrefix = " span_id="; + char traceId[kTraceIdHexLen], spanId[kSpanIdHexLen]; + spanCtx.trace_id().ToLowerBase16( + opentelemetry::nostd::span{traceId}); + spanCtx.span_id().ToLowerBase16( + opentelemetry::nostd::span{spanId}); + output += kTraceIdPrefix; + output.append(traceId, kTraceIdHexLen); + output += kSpanIdPrefix; + output.append(spanId, kSpanIdHexLen); + output += ' '; + } + } + } +#endif // XRPL_ENABLE_TELEMETRY + output += message; // Limit the maximum length of the output diff --git a/src/libxrpl/beast/insight/OTelCollector.cpp b/src/libxrpl/beast/insight/OTelCollector.cpp new file mode 100644 index 00000000000..ab171592720 --- /dev/null +++ b/src/libxrpl/beast/insight/OTelCollector.cpp @@ -0,0 +1,913 @@ +/** + * @file OTelCollector.cpp + * @brief OpenTelemetry Metrics SDK implementation of beast::insight::Collector. + * + * Compiled only when XRPL_ENABLE_TELEMETRY is defined (via CMake + * telemetry=ON). Maps beast::insight instruments to OTel SDK instruments + * created on the GLOBAL Meter published by the telemetry module. This class + * is a legacy shim: it no longer owns an export pipeline. The MeterProvider, + * PeriodicExportingMetricReader, OTLP exporter and histogram view all live in + * xrpl::telemetry::Telemetry. + * + * When XRPL_ENABLE_TELEMETRY is not defined, OTelCollector::New() returns + * a NullCollector so the build succeeds without OTel dependencies. + * + * Data flow: + * + * beast::insight callers + * | + * v + * OTelCounterImpl / OTelGaugeImpl / OTelEventImpl / OTelMeterImpl + * | | | | + * v v v v + * Counter ObservableGauge Histogram Counter + * | | | | + * +--------------------+----------------+--------------+ + * | + * v + * GLOBAL Meter (from metrics::Provider::GetMeterProvider()) + * | + * v + * telemetry-owned PeriodicMetricReader (1s) -> OTLP exporter -> Prometheus + */ + +#ifdef XRPL_ENABLE_TELEMETRY +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace beast::insight { + +namespace detail { + +namespace metrics_api = opentelemetry::metrics; + +class OTelCollectorImp; + +//------------------------------------------------------------------------------ + +/** + * @brief OTel-backed implementation of beast::insight::HookImpl. + * + * Stores a handler function that is invoked during each periodic + * metric collection cycle. This mirrors the StatsDHookImpl pattern + * where hooks are called at each 1-second timer tick, but here the + * invocation is triggered by the OTel PeriodicMetricReader's + * observable callback mechanism. + */ +class OTelHookImpl : public HookImpl +{ +public: + /** + * @param handler Callback invoked at each collection interval. + * @param impl Owning collector (prevents premature destruction). + */ + OTelHookImpl(HandlerType handler, std::shared_ptr impl); + + ~OTelHookImpl() override; + + OTelHookImpl& + operator=(OTelHookImpl const&) = delete; + + /** + * @brief Invoke the stored handler. + * + * Called by the collector during observable gauge callbacks to give + * metric producers a chance to update gauge values before export. + */ + void + callHandler(); + +private: + /** + * Owning collector. Prevents collector destruction while hook alive. + */ + std::shared_ptr impl_; + + /** + * User-supplied handler called at each collection interval. + */ + HandlerType handler_; +}; + +//------------------------------------------------------------------------------ + +/** + * @brief OTel-backed implementation of beast::insight::CounterImpl. + * + * Wraps an OTel Counter instrument. Each increment() call + * is forwarded directly to the OTel counter's Add() method. The + * PeriodicMetricReader collects and exports the accumulated delta. + * + * Thread safety: OTel Counter::Add() is thread-safe by specification. + */ +class OTelCounterImpl : public CounterImpl +{ +public: + /** + * @param name Export-ready metric name, already run through + * formatName() by the collector: prefix prepended and + * dots replaced with underscores (e.g. "rpc_size"). + * @param meter OTel Meter used to create the counter instrument. + */ + OTelCounterImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter); + + ~OTelCounterImpl() override = default; + + OTelCounterImpl& + operator=(OTelCounterImpl const&) = delete; + + /** + * @brief Add amount to the counter. + * @param amount Value to add (must be non-negative for OTel counters). + */ + void + increment(value_type amount) override; + +private: + /** + * OTel synchronous counter instrument. + */ + opentelemetry::nostd::unique_ptr> counter_; +}; + +//------------------------------------------------------------------------------ + +/** + * @brief OTel-backed implementation of beast::insight::EventImpl. + * + * Wraps an OTel Histogram instrument. Each notify() call + * records the duration in milliseconds. Uses explicit bucket boundaries + * matching the SpanMetrics connector configuration: + * [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms + * + * Thread safety: OTel Histogram::Record() is thread-safe by specification. + */ +class OTelEventImpl : public EventImpl +{ +public: + /** + * @param name Export-ready metric name, already run through + * formatName() by the collector: prefix prepended and + * dots replaced with underscores (e.g. "rpc_size"). + * @param meter OTel Meter used to create the histogram instrument. + */ + OTelEventImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter); + + ~OTelEventImpl() override = default; + + OTelEventImpl& + operator=(OTelEventImpl const&) = delete; + + /** + * @brief Record a duration measurement. + * @param value Duration in milliseconds. + */ + void + notify(value_type const& value) override; + +private: + /** + * OTel histogram instrument for recording durations. + */ + opentelemetry::nostd::unique_ptr> histogram_; +}; + +//------------------------------------------------------------------------------ + +/** + * @brief OTel-backed implementation of beast::insight::GaugeImpl. + * + * Uses an atomic int64_t to store the current gauge value. The OTel SDK + * reads this value via an ObservableGauge async callback during each + * collection cycle. The set() and increment() methods update the + * atomic value without blocking the collection thread. + * + * Design note: OTel gauges are asynchronous (observable) instruments. + * The SDK calls a registered callback to read the value rather than + * accepting push-style updates. We bridge the beast::insight push-style + * API to OTel's pull-style API via the atomic variable. + * + * Thread safety: std::atomic operations are lock-free on all platforms. + */ +class OTelGaugeImpl : public GaugeImpl +{ +public: + /** + * @param name Export-ready metric name, already run through + * formatName() by the collector: prefix prepended + * and dots replaced with underscores. + * @param meter OTel Meter used to create the observable gauge. + * @param collector Owning collector, used to invoke hooks before reads. + */ + OTelGaugeImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter, + std::shared_ptr const& collector); + + ~OTelGaugeImpl() override; + + /** + * @brief Set the gauge to an absolute value. + * @param value New gauge value. + */ + void + set(value_type value) override; + + /** + * @brief Increment (or decrement) the gauge by a signed amount. + * + * Clamps the result to [0, INT64_MAX] to match StatsDGaugeImpl + * behavior. + * + * @param amount Signed amount to add to the current value. + */ + void + increment(difference_type amount) override; + + /** + * @brief Return the current gauge value for the OTel callback. + * @return The most recently set/incremented value. + */ + int64_t + currentValue() const; + + OTelGaugeImpl& + operator=(OTelGaugeImpl const&) = delete; + + /** + * Static callback registered with the OTel SDK observable gauge. + */ + static void + gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state); + +private: + /** + * Current gauge value, updated atomically by set()/increment(). + */ + std::atomic value_{0}; + + /** + * OTel observable gauge handle (prevents deregistration). + */ + opentelemetry::nostd::shared_ptr gauge_; + + /** + * Owning collector, used to invoke hooks before reading gauge values. + */ + std::shared_ptr collector_; +}; + +//------------------------------------------------------------------------------ + +/** + * @brief OTel-backed implementation of beast::insight::MeterImpl. + * + * Wraps an OTel Counter instrument. Semantically identical + * to Counter but uses unsigned values. The OTel SDK accumulates deltas + * and exports them via the PeriodicMetricReader. + * + * Note: In StatsD, Meter used the non-standard "|m" type which was + * silently dropped by the OTel StatsD receiver. With native OTel, + * Meter values are properly captured as counter deltas. + * + * Thread safety: OTel Counter::Add() is thread-safe by specification. + */ +class OTelMeterImpl : public MeterImpl +{ +public: + /** + * @param name Export-ready metric name, already run through + * formatName() by the collector: prefix prepended and + * dots replaced with underscores (e.g. "rpc_size"). + * @param meter OTel Meter used to create the counter instrument. + */ + OTelMeterImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter); + + ~OTelMeterImpl() override = default; + + OTelMeterImpl& + operator=(OTelMeterImpl const&) = delete; + + /** + * @brief Add amount to the meter. + * @param amount Value to add (unsigned). + */ + void + increment(value_type amount) override; + +private: + /** + * OTel synchronous counter instrument (unsigned). + */ + opentelemetry::nostd::unique_ptr> counter_; +}; + +//------------------------------------------------------------------------------ + +/** + * @brief Main OTel Collector implementation (legacy shim). + * + * Obtains its Meter from the GLOBAL MeterProvider owned and published by the + * telemetry module (xrpl::telemetry::Telemetry), rather than building its own + * export pipeline. Implements all Collector::make_*() factory methods to + * create OTel-backed instrument wrappers on that shared Meter. + * + * The metrics pipeline (MeterProvider + PeriodicExportingMetricReader + OTLP + * HTTP exporter + histogram view) lives in the telemetry module. This class is + * a thin adapter kept for beast::insight callers during deprecation. + * + * Class diagram: + * + * +------------------+ +------------------+ + * | Collector (ABC) |<-----| OTelCollector | + * +------------------+ | (public header) | + * ^ +------------------+ + * | ^ + * +------------------+ | + * | OTelCollectorImp |-------------+ + * +------------------+ + * | - journal_ | + * | - prefix_ | +--------------------------+ + * | - otelMeter_ |---->| GLOBAL MeterProvider | + * | - hooks_[] | | (owned by Telemetry) | + * | - gauges_[] | | + PeriodicReader | + * +------------------+ | + OtlpHttpExporter | + * +--------------------------+ + * + * Lifecycle: + * 1. Constructor fetches the Meter from the global MeterProvider. + * 2. make_*() methods create instruments on that shared Meter. + * 3. The telemetry-owned PeriodicMetricReader collects every 1s, calling + * observable callbacks. + * 4. Observable callbacks invoke hooks, read gauge atomics. + * 5. Destructor only logs; the telemetry module owns pipeline teardown. + * + * Caveats: + * - Observable gauge callbacks run on the SDK's internal thread. Hook + * handlers must be thread-safe. + * - Metric names are formed as "prefix_name" with dots replaced by + * underscores to match StatsD->Prometheus naming conventions. + * - The OTel Prometheus exporter appends "_total" to counters. The + * metric names we register do NOT include this suffix — Prometheus + * adds it automatically. + * + * Example usage: + * @code + * auto collector = OTelCollector::New( + * "http://localhost:4318/v1/metrics", "xrpld", + * "node-1", "xrpld", "mainnet", journal); + * auto counter = collector->makeCounter("rpc.requests"); + * counter.increment(1); + * // Metric "rpc_requests" exported via OTLP every 1s. + * @endcode + */ +class OTelCollectorImp : public OTelCollector, public std::enable_shared_from_this +{ +public: + /** + * @brief Construct the OTel collector over the global MeterProvider. + * + * @param endpoint OTLP/HTTP metrics endpoint URL. Informational only: + * the global telemetry pipeline is authoritative for + * the actual export endpoint. Retained for logging and + * back-compat with the New() signature. + * @param prefix Prefix for all metric names. + * @param instanceId Value for the service.instance.id resource attribute. + * When empty, the attribute is omitted. + * @param serviceName Value for the service.name resource attribute. + * When empty, defaults to "xrpld". + * @param networkType Value for the xrpl.network.type resource attribute. + * When empty, the attribute is omitted. + * @param journal Journal for logging. + */ + OTelCollectorImp( + std::string const& endpoint, + std::string prefix, + std::string const& instanceId, + std::string const& serviceName, + std::string const& networkType, + Journal journal); + + /** + * @brief Shut down the MeterProvider, flushing any pending exports. + */ + ~OTelCollectorImp() override; + + /** + * @name Collector interface implementation + */ + /** @{ */ + Hook + makeHook(HookImpl::HandlerType const& handler) override; + + Counter + makeCounter(std::string const& name) override; + + Event + makeEvent(std::string const& name) override; + + Gauge + makeGauge(std::string const& name) override; + + Meter + makeMeter(std::string const& name) override; + /** @} */ + + /** + * @name Hook management for observable callbacks + */ + /** @{ */ + + /** + * @brief Register a hook for periodic invocation. + * @param hook Pointer to the hook to register. + */ + void + addHook(OTelHookImpl* hook); + + /** + * @brief Unregister a hook. + * @param hook Pointer to the hook to unregister. + */ + void + removeHook(OTelHookImpl* hook); + + /** + * @brief Invoke all registered hooks. + * + * Called from observable gauge callbacks before reading gauge values, + * so that hook handlers have a chance to update metrics. + */ + void + callHooks(); + /** @} */ + + /** + * @name Gauge registration for observable callbacks + */ + /** @{ */ + + /** + * @brief Register a gauge for observable callback reading. + * @param gauge Pointer to the gauge to register. + */ + void + addGauge(OTelGaugeImpl* gauge); + + /** + * @brief Unregister a gauge. + * @param gauge Pointer to the gauge to unregister. + */ + void + removeGauge(OTelGaugeImpl* gauge); + /** @} */ + + /** + * @brief Format a metric name with the configured prefix. + * + * Replaces dots with underscores to match StatsD->Prometheus naming. + * Example: prefix="xrpld", name="LedgerMaster.Validated_Ledger_Age" + * -> "ledgermaster_validated_ledger_age" + * + * @param name Raw metric name from beast::insight callers. + * @return Fully-qualified metric name. + */ + static std::string + formatName(std::string const& name); + +private: + /** + * Journal for log output. + */ + Journal journal_; + + /** + * Prefix for all metric names (e.g., "xrpld"). + */ + std::string prefix_; + + /** + * OTel Meter used to create all instruments. + */ + opentelemetry::nostd::shared_ptr otelMeter_; + + /** + * Mutex protecting hook and gauge registration lists. + */ + std::mutex mutex_; + + /** + * Registered hooks called during observable callbacks. + */ + std::vector hooks_; + + /** + * Registered gauges read during observable callbacks. + */ + std::vector gauges_; + + /** + * @brief Debounce timestamp for callHooks(). + * + * Multiple gauge callbacks fire during the same collection cycle. + * This atomic tracks the last time hooks were invoked (ms since epoch). + * Hooks are called at most once per 500ms window to avoid redundant + * invocations while still ensuring fresh values each collection cycle. + */ + std::atomic lastHookCallMs_{0}; +}; + +//============================================================================== +// Implementation +//============================================================================== + +//------------------------------------------------------------------------------ +// OTelHookImpl +//------------------------------------------------------------------------------ + +OTelHookImpl::OTelHookImpl(HandlerType handler, std::shared_ptr impl) + : impl_(std::move(impl)), handler_(std::move(handler)) +{ + impl_->addHook(this); +} + +OTelHookImpl::~OTelHookImpl() +{ + impl_->removeHook(this); +} + +void +OTelHookImpl::callHandler() +{ + handler_(); +} + +//------------------------------------------------------------------------------ +// OTelCounterImpl +//------------------------------------------------------------------------------ + +OTelCounterImpl::OTelCounterImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter) + : counter_(meter->CreateUInt64Counter(name)) +{ +} + +void +OTelCounterImpl::increment(value_type amount) +{ + // OTel counters require non-negative values. beast::insight CounterImpl + // uses int64_t, so clamp negative values to 0 and cast to uint64_t. + if (amount > 0) + counter_->Add(static_cast(amount)); +} + +//------------------------------------------------------------------------------ +// OTelEventImpl +//------------------------------------------------------------------------------ + +OTelEventImpl::OTelEventImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter) + : histogram_(meter->CreateDoubleHistogram(name, "Duration in ms", "ms")) +{ +} + +void +OTelEventImpl::notify(value_type const& value) +{ + histogram_->Record(static_cast(value.count()), opentelemetry::context::Context{}); +} + +//------------------------------------------------------------------------------ +// OTelGaugeImpl +//------------------------------------------------------------------------------ + +OTelGaugeImpl::OTelGaugeImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter, + std::shared_ptr const& collector) + : gauge_(meter->CreateInt64ObservableGauge(name)), collector_(collector) +{ + collector_->addGauge(this); + gauge_->AddCallback(gaugeCallback, this); +} + +void +OTelGaugeImpl::gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state) +{ + auto* self = static_cast(state); + self->collector_->callHooks(); + if (auto intResult = opentelemetry::nostd::get_if< + opentelemetry::nostd::shared_ptr>>( + &result)) + { + (*intResult)->Observe(self->currentValue()); + } +} + +OTelGaugeImpl::~OTelGaugeImpl() +{ + // RemoveCallback must run before this object is destroyed so the SDK + // collection thread cannot invoke gaugeCallback on a dangling `this`. + // The SDK's ObservableRegistry guards its callback list and the Observe() + // pass with the same mutex, so RemoveCallback cannot return while a + // callback for this instrument is in flight — removal is synchronous. + gauge_->RemoveCallback(gaugeCallback, this); + collector_->removeGauge(this); +} + +void +OTelGaugeImpl::set(value_type value) +{ + value_.store(static_cast(value), std::memory_order_relaxed); +} + +void +OTelGaugeImpl::increment(difference_type amount) +{ + // Use compare-exchange loop to safely clamp to [0, MAX]. + int64_t current = value_.load(std::memory_order_relaxed); + int64_t desired = 0; + do + { + desired = current + amount; + // Clamp to 0 on underflow. + desired = std::max(desired, int64_t{0}); + } while (!value_.compare_exchange_weak(current, desired, std::memory_order_relaxed)); +} + +int64_t +OTelGaugeImpl::currentValue() const +{ + return value_.load(std::memory_order_relaxed); +} + +//------------------------------------------------------------------------------ +// OTelMeterImpl +//------------------------------------------------------------------------------ + +OTelMeterImpl::OTelMeterImpl( + std::string const& name, + opentelemetry::nostd::shared_ptr const& meter) + : counter_(meter->CreateUInt64Counter(name)) +{ +} + +void +OTelMeterImpl::increment(value_type amount) +{ + counter_->Add(amount); +} + +//------------------------------------------------------------------------------ +// OTelCollectorImp +//------------------------------------------------------------------------------ + +OTelCollectorImp::OTelCollectorImp( + std::string const& endpoint, + std::string prefix, + std::string const& instanceId, + std::string const& serviceName, + std::string const& networkType, + Journal journal) + : journal_(journal), prefix_(std::move(prefix)) +{ + // instanceId/serviceName/networkType are retained on the New() signature + // for back-compat but no longer used here: the telemetry module owns the + // resource attributes for the shared metrics pipeline. + (void)instanceId; + (void)serviceName; + (void)networkType; + + if (journal_.info()) + { + // endpoint is informational: the global telemetry pipeline owns the + // real exporter. It is logged here for back-compat and diagnostics. + journal_.info() << "OTelCollector starting: endpoint=" << endpoint << " prefix=" << prefix_; + } + + // Fetch the Meter from the GLOBAL MeterProvider. The telemetry module + // (xrpl::telemetry::Telemetry) builds the metrics pipeline (exporter, + // periodic reader, histogram view, resource attributes) and registers it + // via metrics::Provider::SetMeterProvider() during start(). beast metrics + // ride that shared pipeline, so both direct-API and beast-sourced metrics + // export under one resource identity. + // + // The name/version literals MUST match the telemetry module's kMeterName + // ("xrpld") and kMeterVersion ("1.0.0"). They are written as literals (not + // referenced from Telemetry.h) because beast/insight sits below the + // telemetry module in the layering and cannot include its header. + otelMeter_ = metrics_api::Provider::GetMeterProvider()->GetMeter( + std::string{"xrpld"}, std::string{"1.0.0"}); + + if (journal_.info()) + { + journal_.info() << "OTelCollector started successfully"; + } +} + +OTelCollectorImp::~OTelCollectorImp() +{ + if (journal_.info()) + { + journal_.info() << "OTelCollector shutting down"; + } + // No pipeline teardown here: the telemetry module owns the global + // MeterProvider lifecycle (ForceFlush/Shutdown happen in Telemetry::stop()). + if (journal_.info()) + { + journal_.info() << "OTelCollector stopped"; + } +} + +Hook +OTelCollectorImp::makeHook(HookImpl::HandlerType const& handler) +{ + return Hook(std::make_shared(handler, shared_from_this())); +} + +Counter +OTelCollectorImp::makeCounter(std::string const& name) +{ + return Counter(std::make_shared(formatName(name), otelMeter_)); +} + +Event +OTelCollectorImp::makeEvent(std::string const& name) +{ + return Event(std::make_shared(formatName(name), otelMeter_)); +} + +Gauge +OTelCollectorImp::makeGauge(std::string const& name) +{ + return Gauge(std::make_shared(formatName(name), otelMeter_, shared_from_this())); +} + +Meter +OTelCollectorImp::makeMeter(std::string const& name) +{ + return Meter(std::make_shared(formatName(name), otelMeter_)); +} + +void +OTelCollectorImp::addHook(OTelHookImpl* hook) +{ + std::scoped_lock const lock(mutex_); + hooks_.push_back(hook); +} + +void +OTelCollectorImp::removeHook(OTelHookImpl* hook) +{ + std::scoped_lock const lock(mutex_); + std::erase(hooks_, hook); +} + +void +OTelCollectorImp::callHooks() +{ + // Debounce: hooks run at most once per 500ms. Multiple gauge callbacks + // fire during the same collection cycle — only the first one triggers + // hooks. Subsequent callbacks within the window read already-updated + // gauge values. + auto now = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + auto last = lastHookCallMs_.load(std::memory_order_acquire); + if (now - last < 500) + return; + if (!lastHookCallMs_.compare_exchange_strong(last, now, std::memory_order_acq_rel)) + return; // Another thread won the race. + + // Copy the hook list under the lock, then invoke handlers outside it. + // A handler may drop the last reference to an OTelHookImpl, whose + // destructor calls removeHook() and re-acquires mutex_; invoking + // handlers while holding the (non-recursive) lock would deadlock. + std::vector hooks; + { + std::scoped_lock const lock(mutex_); + hooks = hooks_; + } + for (auto* hook : hooks) + hook->callHandler(); +} + +void +OTelCollectorImp::addGauge(OTelGaugeImpl* gauge) +{ + std::scoped_lock const lock(mutex_); + gauges_.push_back(gauge); +} + +void +OTelCollectorImp::removeGauge(OTelGaugeImpl* gauge) +{ + std::scoped_lock const lock(mutex_); + std::erase(gauges_, gauge); +} + +std::string +OTelCollectorImp::formatName(std::string const& name) +{ + // Produce a clean, lowercase, Prometheus-compatible metric name. + // No prefix — the OTel resource (service.name) identifies the service. + // Dots and spaces become underscores; everything lowercased. + std::string result; + result.reserve(name.size()); + for (char const c : name) + { + if (c == '.' || c == ' ') + { + result += '_'; + } + else + { + result += static_cast(std::tolower(static_cast(c))); + } + } + return result; +} + +} // namespace detail + +//------------------------------------------------------------------------------ + +std::shared_ptr +OTelCollector::New( + std::string const& endpoint, + std::string const& prefix, + std::string const& instanceId, + std::string const& serviceName, + std::string const& networkType, + Journal journal) +{ + return std::make_shared( + endpoint, prefix, instanceId, serviceName, networkType, journal); +} + +} // namespace beast::insight + +#else // !XRPL_ENABLE_TELEMETRY + +// When telemetry is disabled at compile time, OTelCollector::New() +// returns a NullCollector so callers do not need conditional logic. + +#include +#include +#include +#include + +namespace beast::insight { + +std::shared_ptr +OTelCollector::New( + std::string const& /* endpoint */, + std::string const& /* prefix */, + std::string const& /* instanceId */, + std::string const& /* serviceName */, + std::string const& /* networkType */, + Journal /* journal */) +{ + return NullCollector::make(); +} + +} // namespace beast::insight + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/libxrpl/beast/insight/StatsDCollector.cpp b/src/libxrpl/beast/insight/StatsDCollector.cpp index 3cff5d93b5c..9a14393a5e8 100644 --- a/src/libxrpl/beast/insight/StatsDCollector.cpp +++ b/src/libxrpl/beast/insight/StatsDCollector.cpp @@ -166,7 +166,7 @@ class StatsDGaugeImpl : public GaugeImpl, public StatsDMetricBase std::string name_; GaugeImpl::value_type lastValue_{0}; GaugeImpl::value_type value_{0}; - bool dirty_{false}; + bool dirty_{true}; }; //------------------------------------------------------------------------------ @@ -572,6 +572,9 @@ StatsDEventImpl::doNotify(EventImpl::value_type const& value) StatsDGaugeImpl::StatsDGaugeImpl(std::string name, std::shared_ptr impl) : impl_(std::move(impl)), name_(std::move(name)) { + // Start dirty so the initial value (0) is emitted on the first flush. + // Without this, gauges whose value never changes from 0 would never + // appear in downstream metric stores (e.g. Prometheus via StatsD). impl_->add(*this); } diff --git a/src/libxrpl/core/detail/Job.cpp b/src/libxrpl/core/detail/Job.cpp index 89fe42db051..70c11b3bf19 100644 --- a/src/libxrpl/core/detail/Job.cpp +++ b/src/libxrpl/core/detail/Job.cpp @@ -36,6 +36,12 @@ Job::getType() const return type_; } +std::string const& +Job::getName() const +{ + return name_; +} + Job::clock_type::time_point const& Job::queueTime() const { diff --git a/src/libxrpl/core/detail/JobQueue.cpp b/src/libxrpl/core/detail/JobQueue.cpp index 8f95a8a5fa1..71051839a96 100644 --- a/src/libxrpl/core/detail/JobQueue.cpp +++ b/src/libxrpl/core/detail/JobQueue.cpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace xrpl { @@ -65,8 +66,45 @@ JobQueue::~JobQueue() void JobQueue::collect() { - std::scoped_lock const lock(mutex_); - jobCount_ = jobSet_.size(); + // Gauge::value_type is unsigned. The counters are only ever expected to + // be non-negative (the surrounding asserts enforce it), but clamp anyway: + // an unclamped negative would wrap to ~1.8e19 and swamp every dashboard + // reading this family. + auto const toGauge = [](int value) { + return static_cast(std::max(0, value)); + }; + + // Snapshot under the lock, publish after releasing it. + // + // Writing gauges while holding `mutex_` would invert a lock order: a + // collector's gauge write can take the collector's own lock, while the + // collector's flush thread already holds that lock when it calls this + // hook and then needs `mutex_`. Publishing outside the lock keeps + // `mutex_` strictly innermost, so no cycle can form. It also keeps the + // queue's hot lock (every addJob and every job start/finish takes it) + // off the per-gauge write path, which on some collectors allocates. + // + // A snapshot can be one flush interval stale; these are sampled + // saturation gauges, so that is the intended accuracy. + std::vector> snapshot; + { + std::scoped_lock const lock(mutex_); + jobCount_ = jobSet_.size(); + + snapshot.reserve(jobData_.size()); + for (auto& [type, data] : jobData_) + snapshot.emplace_back(&data, data.waiting, data.running, data.deferred); + } + + // Gauges exist only for non-special types (JobTypeData's ctor). Assigning + // to a special type's default-constructed Gauge is a safe no-op -- + // Gauge::set() checks its impl pointer -- so this needs no special case. + for (auto const& [data, waiting, running, deferred] : snapshot) + { + data->waitingGauge = toGauge(waiting); + data->runningGauge = toGauge(running); + data->deferredGauge = toGauge(deferred); + } } bool @@ -98,7 +136,9 @@ JobQueue::addRefCountedJob(JobType type, std::string const& name, JobFunction co JobType const type(job.getType()); XRPL_ASSERT(type != JtInvalid, "xrpl::JobQueue::addRefCountedJob : has valid job type"); XRPL_ASSERT(jobSet_.contains(job), "xrpl::JobQueue::addRefCountedJob : job found"); - perfLog_.jobQueue(type); + // `name` is the addJob name; it becomes the `handler` metric label so + // producers sharing a job type stay individually attributable. + perfLog_.jobQueue(type, name); JobTypeData& data(getJobTypeData(type)); @@ -359,7 +399,7 @@ JobQueue::processTask(int instance) // The amount of time that the job was in the queue auto const qTime = ceil(startTime - job.queueTime()); - perfLog_.jobStart(type, qTime, startTime, instance); + perfLog_.jobStart(type, job.getName(), qTime, startTime, instance); job.doJob(); @@ -371,7 +411,9 @@ JobQueue::processTask(int instance) getJobTypeData(type).dequeue.notify(qTime); getJobTypeData(type).execute.notify(xTime); } - perfLog_.jobFinish(type, xTime, instance); + // `job` is still alive here: it is scoped to the enclosing block + // and doJob() only releases the callable, not the name. + perfLog_.jobFinish(type, job.getName(), xTime, instance); } } diff --git a/src/libxrpl/telemetry/CoroAwareContextStorage.cpp b/src/libxrpl/telemetry/CoroAwareContextStorage.cpp new file mode 100644 index 00000000000..f3d13c64890 --- /dev/null +++ b/src/libxrpl/telemetry/CoroAwareContextStorage.cpp @@ -0,0 +1,84 @@ +/** + * Implementation of CoroAwareContextStorage. + * + * Mirrors opentelemetry's ThreadLocalContextStorage stack semantics, but the + * stack lives in an xrpl::LocalValue so it follows a JobQueue::Coro across + * yield/resume. All OpenTelemetry types stay confined to this translation unit. + * + * @see CoroAwareContextStorage (CoroAwareContextStorage.h) + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include + +#include + +#include +#include +#include + +namespace xrpl::telemetry { + +opentelemetry::context::Context +CoroAwareContextStorage::GetCurrent() noexcept +{ + // Hot path: Log.cpp reads the current context on EVERY log line on EVERY + // thread. Peek the raw per-thread/coro store WITHOUT materializing it: + // getLocalValues().get() returns nullptr when this thread has no store yet + // and never allocates. Dereferencing stack_ (a LocalValue) would instead + // heap-allocate the store + a map entry + the vector on first touch, so we + // avoid that on the no-span path. + // + // Behaviour is identical to touching *stack_: a thread with no store has no + // active span, and a store whose stack was never pushed yields an empty + // vector — both cases map to an empty Context, matching stock + // ThreadLocalContextStorage (Top() returns Context() when the stack is + // empty). Only when a store already exists do we read the top frame. + if (detail::getLocalValues().get() == nullptr) + return opentelemetry::context::Context{}; + auto& stack = *stack_; + return stack.empty() ? opentelemetry::context::Context{} : stack.back(); +} + +opentelemetry::nostd::unique_ptr +CoroAwareContextStorage::Attach(opentelemetry::context::Context const& context) noexcept +{ + stack_->push_back(context); + return CreateToken(context); +} + +bool +CoroAwareContextStorage::Detach(opentelemetry::context::Token& token) noexcept +{ + auto& stack = *stack_; + // Fast path: the token's frame is on top (the common LIFO case). + if (!stack.empty() && token == stack.back()) + { + stack.pop_back(); + return true; + } + // Fallback: token not on top — verify it is present, then pop down to and + // including it (mirrors ThreadLocalContextStorage::Detach; also detaches + // any child frames left above it). + bool found = false; + for (auto const& frame : stack) + { + if (token == frame) + { + found = true; + break; + } + } + if (!found) + return false; + while (!stack.empty() && !(token == stack.back())) + stack.pop_back(); + if (!stack.empty()) + stack.pop_back(); + return true; +} + +} // namespace xrpl::telemetry + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/libxrpl/telemetry/DeterministicIdGenerator.cpp b/src/libxrpl/telemetry/DeterministicIdGenerator.cpp new file mode 100644 index 00000000000..90c5c744b3b --- /dev/null +++ b/src/libxrpl/telemetry/DeterministicIdGenerator.cpp @@ -0,0 +1,117 @@ +/** + * Implementation of DeterministicIdGenerator and PendingTraceId. + * + * The pending trace_id lives in file-local thread_locals shared between the + * generator and the RAII guard. GenerateTraceId() consumes the pending id on + * the SDK's no-parent branch; PendingTraceId sets it and asserts consumption. + * All OpenTelemetry SDK types stay confined to telemetry translation units. + * + * @see DeterministicIdGenerator, PendingTraceId (DeterministicIdGenerator.h) + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace xrpl::telemetry { + +namespace { + +/** + * Trace_id pinned by PendingTraceId, consumed by GenerateTraceId(). File-local + * and thread-local, so it is set and read on the same thread with no locking. + */ +thread_local std::optional> gTlsPendingTraceId; + +/** + * True once GenerateTraceId() has consumed the pending id. ~PendingTraceId + * asserts on it to catch a forced-root span that never reached the SDK root + * branch. + */ +thread_local bool gTlsPendingConsumed = false; + +/** + * Process-wide count of deterministic trace_ids dropped without being consumed. + * Bumped in ~PendingTraceId when the pending id was never taken by a root span. + * Atomic because it is summed across all threads; relaxed ordering is enough + * for a diagnostic tally. Exposed via unconsumedDeterministicIdDrops(). + */ +std::atomic gUnconsumedDeterministicIdDrops{0}; + +} // namespace + +DeterministicIdGenerator::DeterministicIdGenerator() + : opentelemetry::sdk::trace::IdGenerator(/*is_random=*/false) + , random_(opentelemetry::sdk::trace::RandomIdGeneratorFactory::Create()) +{ +} + +opentelemetry::trace::TraceId +DeterministicIdGenerator::GenerateTraceId() noexcept +{ + if (gTlsPendingTraceId) + { + auto const id = *gTlsPendingTraceId; + gTlsPendingTraceId.reset(); + gTlsPendingConsumed = true; + return opentelemetry::trace::TraceId( + opentelemetry::nostd::span(id.data(), 16)); + } + return random_->GenerateTraceId(); +} + +opentelemetry::trace::SpanId +DeterministicIdGenerator::GenerateSpanId() noexcept +{ + return random_->GenerateSpanId(); // ALWAYS random — never uses the pending id +} + +PendingTraceId::PendingTraceId(std::array const& id) noexcept +{ + gTlsPendingTraceId = id; + gTlsPendingConsumed = false; +} + +PendingTraceId::~PendingTraceId() noexcept +{ + // The forced no-parent (root) span-start MUST have consumed the pending id + // via GenerateTraceId(). If not, the SDK took a different branch than the + // caller forced — a bug — so fail loudly in debug/test builds. + XRPL_ASSERT( + gTlsPendingConsumed, + "xrpl::telemetry::PendingTraceId : deterministic trace_id was not consumed"); + if (!gTlsPendingConsumed) + { + // The assert above is stripped in release, so bump a counter too: a + // process-wide tally of dropped deterministic trace roots. It is + // exposed via unconsumedDeterministicIdDrops() as an extension point + // for a future metric or diagnostic to read; nothing wires it to a log + // or metric yet. Relaxed: this is a plain diagnostic tally. + gUnconsumedDeterministicIdDrops.fetch_add(1, std::memory_order_relaxed); + } + gTlsPendingTraceId.reset(); // never leak, even in release + gTlsPendingConsumed = false; +} + +std::uint64_t +unconsumedDeterministicIdDrops() noexcept +{ + return gUnconsumedDeterministicIdDrops.load(std::memory_order_relaxed); +} + +} // namespace xrpl::telemetry + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/libxrpl/telemetry/NullTelemetry.cpp b/src/libxrpl/telemetry/NullTelemetry.cpp new file mode 100644 index 00000000000..92cc7303efd --- /dev/null +++ b/src/libxrpl/telemetry/NullTelemetry.cpp @@ -0,0 +1,149 @@ +/** + * No-op implementation of the Telemetry interface. + * + * Always compiled (regardless of XRPL_ENABLE_TELEMETRY). Provides the + * makeTelemetry() factory when telemetry is compiled out (#ifndef), which + * unconditionally returns a NullTelemetry that does nothing. + * + * When XRPL_ENABLE_TELEMETRY IS defined, the OTel virtual methods + * (getTracer, startSpan) return noop tracers/spans. The makeTelemetry() + * factory in this file is not used in that case -- Telemetry.cpp provides + * its own factory that can return the real TelemetryImpl. + */ + +#include + +#ifdef XRPL_ENABLE_TELEMETRY +#include +#include +#include +#include +#include +#include + +#include +#endif + +#include +#include +#include + +namespace xrpl::telemetry { + +namespace { + +/** + * No-op Telemetry that returns immediately from every method. + * + * Used as the sole implementation when XRPL_ENABLE_TELEMETRY is not + * defined, or as a fallback when it is defined but enabled=0. + */ +class NullTelemetry : public Telemetry +{ + /** + * Retained configuration (unused, kept for diagnostic access). + */ + Setup const setup_; + +public: + explicit NullTelemetry(Setup setup) : setup_{std::move(setup)} + { + } + + void + start() override + { + Telemetry::setInstance(this); + } + + void + stop() override + { + Telemetry::setInstance(nullptr); + } + + [[nodiscard]] bool + isEnabled() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceTransactions() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceConsensus() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceRpc() const override + { + return false; + } + + [[nodiscard]] bool + shouldTracePeer() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceLedger() const override + { + return false; + } + + [[nodiscard]] std::string const& + getConsensusTraceStrategy() const override + { + return setup_.consensusTraceStrategy; + } + +#ifdef XRPL_ENABLE_TELEMETRY + opentelemetry::nostd::shared_ptr + getTracer(std::string_view) override + { + static auto noopTracer = opentelemetry::nostd::shared_ptr( + new opentelemetry::trace::NoopTracer()); + return noopTracer; + } + + opentelemetry::nostd::shared_ptr + startSpan(std::string_view, opentelemetry::trace::SpanKind) override + { + return opentelemetry::nostd::shared_ptr( + new opentelemetry::trace::NoopSpan(nullptr)); + } + + opentelemetry::nostd::shared_ptr + startSpan( + std::string_view, + opentelemetry::context::Context const&, + opentelemetry::trace::SpanKind) override + { + return opentelemetry::nostd::shared_ptr( + new opentelemetry::trace::NoopSpan(nullptr)); + } +#endif +}; + +} // namespace + +/** + * Factory used when XRPL_ENABLE_TELEMETRY is not defined. + * Unconditionally returns a NullTelemetry instance. + */ +#ifndef XRPL_ENABLE_TELEMETRY +std::unique_ptr +makeTelemetry(Telemetry::Setup const& setup, beast::Journal) +{ + return std::make_unique(setup); +} +#endif + +} // namespace xrpl::telemetry diff --git a/src/libxrpl/telemetry/Redaction.cpp b/src/libxrpl/telemetry/Redaction.cpp new file mode 100644 index 00000000000..7164c4dee62 --- /dev/null +++ b/src/libxrpl/telemetry/Redaction.cpp @@ -0,0 +1,31 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace xrpl::telemetry { + +std::string +redactAccount(std::string_view addr) +{ + // Empty in, empty out: nothing to hash and no token to emit. + if (addr.empty()) + return {}; + + // sha512Half yields a uint256; to_string renders it as uppercase hex. + // Keep the first 16 chars (64 bits) — enough to correlate spans while + // staying non-reversible — and lowercase them for a stable token. + auto const digest = sha512Half(Slice(addr.data(), addr.size())); + std::string token = to_string(digest).substr(0, 16); + std::ranges::transform( + token, token.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + return token; +} + +} // namespace xrpl::telemetry diff --git a/src/libxrpl/telemetry/SpanGuard.cpp b/src/libxrpl/telemetry/SpanGuard.cpp new file mode 100644 index 00000000000..ca5a8142608 --- /dev/null +++ b/src/libxrpl/telemetry/SpanGuard.cpp @@ -0,0 +1,863 @@ +/** + * Pimpl implementation for SpanGuard, ScopedSpanGuard and SpanContext. + * + * All OpenTelemetry SDK types are confined to this translation unit. + * The public SpanGuard.h header contains only standard-library types + * and forward-declares the Impl / ScopedImpl structs. + * + * Two guards with split responsibilities live here: + * - SpanGuard::Impl holds ONLY the OTel Span (shared_ptr). It never + * touches the thread-local context stack, so a SpanGuard is + * thread-free and may be moved to and destroyed on any thread. + * - ScopedSpanGuard::ScopedImpl holds a SpanGuard plus an optional + * Scope. The Scope pushes the span onto the constructing context + * store's stack; member order (guard first, scope second) ensures + * the Scope pops BEFORE the span ends on destruction. It is + * store-bound: construct and destroy while the same LocalValue + * context store is active (the same thread, or the same JobQueue + * coroutine even if it resumes on another worker). + * + * Static factory methods access the global Telemetry instance via + * Telemetry::getInstance(), check whether the requested TraceCategory + * is enabled, and return either an active guard with a real Span or a + * null guard whose methods are all no-ops. + * + * @see SpanGuard, ScopedSpanGuard (SpanGuard.h), Telemetry (Telemetry.h), + * FilteringSpanProcessor (Telemetry.cpp) + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::telemetry { + +namespace otel_trace = opentelemetry::trace; + +// ===== SpanContext::Impl =================================================== + +struct SpanContext::Impl +{ + opentelemetry::context::Context ctx; + + explicit Impl(opentelemetry::context::Context c) : ctx(std::move(c)) + { + } +}; + +SpanContext::SpanContext(std::shared_ptr impl) : impl_(std::move(impl)) +{ +} + +bool +SpanContext::isValid() const noexcept +{ + return impl_ != nullptr; +} + +// ===== SpanGuard::Impl ==================================================== + +struct SpanGuard::Impl +{ + /** + * The OTel span being guarded. Set to nullptr after discard() so + * ~Impl skips End(). + */ + opentelemetry::nostd::shared_ptr span; + + /** + * Construct an unscoped guard: the span is owned but NOT pushed + * onto any thread's context stack. The guard is thread-free. + * @param s The span to guard. + */ + explicit Impl(opentelemetry::nostd::shared_ptr s) : span(std::move(s)) + { + } + + ~Impl() + { + if (span) + span->End(); + } + + Impl(Impl const&) = delete; + Impl& + operator=(Impl const&) = delete; + Impl(Impl&&) = delete; + Impl& + operator=(Impl&&) = delete; +}; + +// ===== SpanGuard core lifecycle ============================================ + +SpanGuard::SpanGuard() = default; +SpanGuard::~SpanGuard() = default; +SpanGuard::SpanGuard(SpanGuard&&) noexcept = default; +SpanGuard& +SpanGuard::operator=(SpanGuard&&) noexcept = default; + +SpanGuard::SpanGuard(std::unique_ptr impl) noexcept : impl_(std::move(impl)) +{ +} + +SpanGuard:: +operator bool() const noexcept +{ + return impl_ != nullptr; +} + +// ===== Static factory methods ============================================== + +/** + * Check whether the given TraceCategory is enabled on the Telemetry instance. + * @return true if the category's shouldTrace*() flag is on. + */ +static bool +isCategoryEnabled(Telemetry const& tel, TraceCategory cat) +{ + switch (cat) + { + case TraceCategory::Rpc: + return tel.shouldTraceRpc(); + case TraceCategory::Transactions: + return tel.shouldTraceTransactions(); + case TraceCategory::Consensus: + return tel.shouldTraceConsensus(); + case TraceCategory::Peer: + return tel.shouldTracePeer(); + case TraceCategory::Ledger: + return tel.shouldTraceLedger(); + } + return false; // unreachable, silences compiler warning +} + +namespace { + +// Span-link attribute marking a "follows-from" (causal, non-parent) link, +// emitted by both linkedSpan() overloads. Custom xrpl attribute — not part +// of the OTel semantic conventions, so defined here rather than pulled from +// . +constexpr char const* kLinkTypeKey = "link_type"; +constexpr char const* kLinkTypeFollowsFrom = "follows_from"; + +// Map a TraceCategory to an OTel SpanKind so Tempo's service-graph / +// RED metrics see the correct direction. RPC spans are emitted at the +// server entry point (handler dispatch), Peer spans at inbound-message +// receipt. Transactions / Consensus / Ledger are internal processing +// and keep the default kInternal. +otel_trace::SpanKind +categoryToSpanKind(TraceCategory cat) +{ + switch (cat) + { + case TraceCategory::Rpc: + return otel_trace::SpanKind::kServer; + case TraceCategory::Peer: + return otel_trace::SpanKind::kConsumer; + case TraceCategory::Transactions: + case TraceCategory::Consensus: + case TraceCategory::Ledger: + return otel_trace::SpanKind::kInternal; + } + return otel_trace::SpanKind::kInternal; // unreachable +} + +} // namespace + +SpanGuard +SpanGuard::span(TraceCategory cat, std::string_view prefix, std::string_view name) noexcept +{ + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled() || !isCategoryEnabled(*tel, cat)) + return {}; + std::string fullName; + fullName.reserve(prefix.size() + 1 + name.size()); + fullName.append(prefix).append(1, '.').append(name); + return SpanGuard(std::make_unique(tel->startSpan(fullName, categoryToSpanKind(cat)))); +} + +SpanGuard +SpanGuard::freshRoot(TraceCategory cat, std::string_view prefix, std::string_view name) noexcept +{ + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled() || !isCategoryEnabled(*tel, cat)) + return {}; + std::string fullName; + fullName.reserve(prefix.size() + 1 + name.size()); + fullName.append(prefix).append(1, '.').append(name); + // Force a fresh trace root: do NOT inherit this thread's active span. + auto rootCtx = opentelemetry::context::Context{otel_trace::kIsRootSpanKey, true}; + return SpanGuard( + std::make_unique(tel->startSpan(fullName, rootCtx, categoryToSpanKind(cat)))); +} + +// ===== Child / linked span creation ======================================== + +SpanGuard +SpanGuard::childSpan(std::string_view name) const noexcept +{ + if (!impl_) + return {}; + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled()) + return {}; + auto ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + return SpanGuard(std::make_unique(tel->startSpan(name, ctx))); +} + +SpanGuard +SpanGuard::childSpan(std::string_view name, SpanContext const& parentCtx) noexcept +{ + if (!parentCtx.isValid()) + return {}; + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled()) + return {}; + return SpanGuard(std::make_unique(tel->startSpan(name, parentCtx.impl_->ctx))); +} + +SpanGuard +SpanGuard::linkedSpan(std::string_view name) const noexcept +{ + if (!impl_) + return {}; + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled()) + return {}; + + auto tracer = tel->getTracer(); + auto spanCtx = impl_->span->GetContext(); + + // Mark as root span so it starts a new trace sub-tree rather than + // inheriting the current thread's active span as parent. + otel_trace::StartSpanOptions opts; + opts.parent = opentelemetry::context::Context{otel_trace::kIsRootSpanKey, true}; + + // LCOV_EXCL_START + return SpanGuard( + std::make_unique(tracer->StartSpan( + std::string(name), {}, {{spanCtx, {{kLinkTypeKey, kLinkTypeFollowsFrom}}}}, opts))); + // LCOV_EXCL_STOP +} + +SpanGuard +SpanGuard::linkedSpan(std::string_view name, SpanContext const& linkCtx) noexcept +{ + if (!linkCtx.isValid()) + return {}; + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled()) + return {}; + + auto tracer = tel->getTracer(); + + // Extract the span from the captured context to get its SpanContext. + auto linkSpan = otel_trace::GetSpan(linkCtx.impl_->ctx); + if (!linkSpan || !linkSpan->GetContext().IsValid()) + return {}; + + // Mark as root span so it starts a new trace sub-tree rather than + // inheriting the current thread's active span as parent. + otel_trace::StartSpanOptions opts; + opts.parent = opentelemetry::context::Context{otel_trace::kIsRootSpanKey, true}; + + // LCOV_EXCL_START + return SpanGuard( + std::make_unique(tracer->StartSpan( + std::string(name), + {}, + {{linkSpan->GetContext(), {{kLinkTypeKey, kLinkTypeFollowsFrom}}}}, + opts))); + // LCOV_EXCL_STOP +} + +// ===== Hash-derived span (category-gated) ================================== + +// Standalone hash-derived span: a TRUE root whose trace_id == hashData[0:16]. +SpanGuard +SpanGuard::hashSpan( + TraceCategory const cat, + std::string_view const name, + std::uint8_t const* const hashData, + std::size_t const hashSize, + SpanContext const* const followsFrom) +{ + if (hashSize < 16) + return {}; + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled() || !isCategoryEnabled(*tel, cat)) + return {}; + + // Force the deterministic trace_id via the DeterministicIdGenerator, on the + // SDK root branch, so the span is a TRUE root (empty parent_span_id) whose + // trace_id == hashData[0:16]. The kIsRootSpanKey context forces the SDK's + // no-valid-parent branch, where GenerateTraceId() is called and returns the + // pending id. The PendingTraceId guard scopes that id to this one startSpan. + std::array tid{}; + std::memcpy(tid.data(), hashData, 16); + auto rootCtx = opentelemetry::context::Context{otel_trace::kIsRootSpanKey, true}; + PendingTraceId const pending{tid}; + + // When a prior context is supplied, attach it as a follows-from LINK (not a + // parent) so consecutive roots (e.g. consensus rounds) stay navigable while + // this span remains a true root under its deterministic trace_id. + if (followsFrom != nullptr && followsFrom->isValid()) + { + auto linkSpan = otel_trace::GetSpan(followsFrom->impl_->ctx); + if (linkSpan && linkSpan->GetContext().IsValid()) + { + auto tracer = tel->getTracer("xrpld"); + otel_trace::StartSpanOptions opts; + opts.parent = rootCtx; // root marker → forces GenerateTraceId() + opts.kind = categoryToSpanKind(cat); + return SpanGuard( + std::make_unique(tracer->StartSpan( + std::string(name), + {}, + {{linkSpan->GetContext(), {{kLinkTypeKey, kLinkTypeFollowsFrom}}}}, + opts))); + } + } + + return SpanGuard( + std::make_unique( + tel->startSpan(std::string(name), rootCtx, categoryToSpanKind(cat)))); +} + +// Cross-node hash-derived span: a CHILD of the sender's real remote span +// (remote=true), not a root — so it must NOT use PendingTraceId / rootCtx. +SpanGuard +SpanGuard::hashSpan( + TraceCategory const cat, + std::string_view const name, + std::uint8_t const* const hashData, + std::size_t const hashSize, + std::uint8_t const* const parentSpanId, + std::size_t const parentSpanSize, + std::uint8_t const traceFlags) +{ + if (hashSize < 16 || parentSpanSize != 8) + return {}; + auto* tel = Telemetry::getInstance(); + if ((tel == nullptr) || !tel->isEnabled() || !isCategoryEnabled(*tel, cat)) + return {}; + + otel_trace::TraceId const traceId( + opentelemetry::nostd::span(hashData, 16)); + + otel_trace::SpanId const parentSpan( + opentelemetry::nostd::span(parentSpanId, 8)); + + otel_trace::SpanContext const combinedCtx( + traceId, + parentSpan, + otel_trace::TraceFlags(traceFlags), + /* remote = */ true); + + auto parentCtx = opentelemetry::context::Context{}.SetValue( + otel_trace::kSpanKey, + opentelemetry::nostd::shared_ptr( + new otel_trace::DefaultSpan(combinedCtx))); + + return SpanGuard(std::make_unique(tel->startSpan(std::string(name), parentCtx))); +} + +// ===== Context capture ===================================================== + +SpanContext +SpanGuard::spanContext() const noexcept +{ + if (!impl_ || !impl_->span) + return {}; + // Refer to THIS guard's own span, not the thread's ambient span. A + // SpanGuard is unscoped (never pushed onto the thread-local stack), so + // RuntimeContext::GetCurrent() would return an unrelated ambient span. + // Building the context from impl_->span makes spanContext() correct + // for every caller regardless of scoping, and lets childSpan(name, ctx) + // parent explicitly to this span across threads. + auto ctx = opentelemetry::context::Context{}.SetValue(otel_trace::kSpanKey, impl_->span); + return SpanContext(std::make_shared(std::move(ctx))); +} + +SpanContext +SpanGuard::threadLocalContext() noexcept +{ + // Snapshot whatever span is currently active on THIS thread's context + // stack (the ambient context), independent of any guard. This is the + // OTel "capture current context" operation used for propagation. + auto ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + return SpanContext(std::make_shared(std::move(ctx))); +} + +TraceBytes +SpanGuard::getTraceBytes() const +{ + if (!impl_ || !impl_->span) + return {}; + + auto const& spanCtx = impl_->span->GetContext(); + if (!spanCtx.IsValid()) + return {}; + + TraceBytes result; + auto const& tid = spanCtx.trace_id(); + std::memcpy(result.traceId.data(), tid.Id().data(), 16); + auto const& sid = spanCtx.span_id(); + std::memcpy(result.spanId.data(), sid.Id().data(), 8); + result.traceFlags = spanCtx.trace_flags().flags(); + result.valid = true; + return result; +} + +void +SpanGuard::injectCurrentContextToProtobuf(protocol::TraceContext& proto) +{ + auto ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + injectToProtobuf(ctx, proto); +} + +// ===== Attribute setters =================================================== + +void +SpanGuard::setAttribute(std::string_view key, std::string_view value) noexcept +{ + if (impl_) + { + impl_->span->SetAttribute( + opentelemetry::nostd::string_view(key.data(), key.size()), + opentelemetry::nostd::string_view(value.data(), value.size())); + } +} + +void +SpanGuard::setAttribute(std::string_view key, char const* value) noexcept +{ + setAttribute(key, std::string_view(value)); +} + +void +SpanGuard::setAttribute(std::string_view key, std::int64_t value) noexcept +{ + if (impl_) + impl_->span->SetAttribute(opentelemetry::nostd::string_view(key.data(), key.size()), value); +} + +void +SpanGuard::setAttribute(std::string_view key, double value) noexcept +{ + if (impl_) + impl_->span->SetAttribute(opentelemetry::nostd::string_view(key.data(), key.size()), value); +} + +void +SpanGuard::setAttribute(std::string_view key, bool value) noexcept +{ + if (impl_) + impl_->span->SetAttribute(opentelemetry::nostd::string_view(key.data(), key.size()), value); +} + +// ===== Status / events ===================================================== + +void +SpanGuard::setOk() noexcept +{ + if (impl_) + impl_->span->SetStatus(otel_trace::StatusCode::kOk); +} + +void +SpanGuard::setError(std::string_view description) noexcept +{ + if (impl_) + impl_->span->SetStatus(otel_trace::StatusCode::kError, std::string(description)); +} + +void +SpanGuard::addEvent(std::string_view name) noexcept +{ + if (impl_) + impl_->span->AddEvent(std::string(name)); +} + +void +SpanGuard::addEvent(std::string_view name, std::initializer_list attrs) noexcept +{ + if (!impl_) + return; + // OTel's AddEvent template requires a key-value-iterable; a plain + // std::vector> doesn't satisfy is_key_value_iterable. + // Wrap in nostd::span over the vector's storage so the SDK accepts it. + std::vector> + + otelAttrs; + otelAttrs.reserve(attrs.size()); + for (auto const& [k, v] : attrs) + { + otelAttrs.emplace_back( + opentelemetry::nostd::string_view{k.data(), k.size()}, + opentelemetry::common::AttributeValue{ + opentelemetry::nostd::string_view{v.data(), v.size()}}); + } + impl_->span->AddEvent( + std::string(name), + opentelemetry::nostd::span const>{otelAttrs.data(), otelAttrs.size()}); +} + +void +SpanGuard::recordException(std::exception const& e) noexcept +{ + if (!impl_) + return; + namespace semconv_exc = opentelemetry::semconv::exception; + // Event name "exception" and the attribute keys follow the OTel semantic + // conventions; the keys come from semconv constants rather than literals. + impl_->span->AddEvent( + "exception", + {{semconv_exc::kExceptionType, typeid(e).name()}, + {semconv_exc::kExceptionMessage, std::string(e.what())}}); + impl_->span->SetStatus(otel_trace::StatusCode::kError, e.what()); +} + +void +SpanGuard::discard() noexcept +{ + if (impl_) + { + { + // DiscardScope owns the flag's whole lifetime: it sets the flag, + // and clears it on scope exit — even if End() were to throw. The + // SDK invokes FilteringSpanProcessor::OnEnd() synchronously from + // End() on this thread, so the flag is observed while still set. + // Today every valid guard wraps a recording span (head sampling is + // 1.0), so OnEnd() always runs — but scoping set/clear keeps the + // flag leak-proof if a later phase can hand back a non-recording + // span (e.g. honoring a non-sampled remote parent during + // propagation), so it can never spill onto the next span. + DiscardScope const discardScope; + impl_->span->End(); + } + impl_->span = nullptr; // prevent ~Impl from calling End() again + impl_.reset(); + } +} + +// ===== ScopedSpanGuard::ScopedImpl ========================================= + +struct ScopedSpanGuard::ScopedImpl +{ + /** + * The unscoped guard that owns the span. Declared FIRST so it is + * destroyed AFTER `scope`: the Scope must pop before the span ends. + */ + SpanGuard guard; + + /** + * Active OTel Scope binding the span to the active context store's + * stack. Present while the guard is active; reset() (via operator + * SpanGuard) pops it eagerly under the constructing store. Declared + * AFTER `guard` so destruction order pops the scope before the span + * ends. + */ + std::optional scope; + + /** + * Identity of the LocalValue store the Scope was pushed onto (the + * coroutine's store on a coro, else the thread's own store). Popping + * the Scope (via ~Scope or reset()) must happen while the SAME store + * is active, else a different stack is corrupted. Coroutine-aware + * storage lets a coro legitimately resume on another worker while + * keeping the same store, so store-identity — not thread-id — is the + * correct invariant. Checked in the destructor and operator + * SpanGuard() to turn a silent cross-store pop into an assertion + * failure in debug/test/fuzzing builds. + * + * Assigned in the constructor body AFTER the Scope push, not as a + * member initializer: the push is the first LocalValue touch on a + * fresh thread for a root or explicit-parent span (the OTel SDK skips + * the ambient-context read in those cases), so it materializes the + * store. Capturing before the push would record nullptr while the + * destructor sees the now-materialized store. For a null guard no + * Scope is pushed and the assertions short-circuit, so this holds + * whatever store is active then. + */ + void const* owner = nullptr; + + /** + * Wrap a SpanGuard and activate its span on this thread. If the + * guard is active, push its span onto the thread-local context + * stack; a null guard leaves `scope` empty. + * @param g The span-owning guard to activate. + */ + explicit ScopedImpl(SpanGuard g) : guard(std::move(g)) + { + if (guard) + scope.emplace(guard.impl_->span); + // Capture after the push so `owner` names the store the Scope + // actually lives on, even when the push materialized it. + owner = detail::getLocalValues().get(); + } +}; + +// ===== ScopedSpanGuard core lifecycle ====================================== + +ScopedSpanGuard::ScopedSpanGuard(SpanGuard&& guard) noexcept + : impl_(std::make_unique(std::move(guard))) +{ +} + +ScopedSpanGuard::~ScopedSpanGuard() +{ + // A live Scope must be popped while its constructing context store is + // active; destroying the guard under a different store corrupts that + // store's context stack. A null guard holds no Scope, so the check is + // skipped. + XRPL_ASSERT( + !impl_ || !impl_->scope.has_value() || impl_->owner == detail::getLocalValues().get(), + "xrpl::telemetry::ScopedSpanGuard::~ScopedSpanGuard : destroyed on the " + "constructing context store"); +} + +// ===== ScopedSpanGuard factory methods ===================================== + +ScopedSpanGuard::ScopedSpanGuard( + TraceCategory cat, + std::string_view prefix, + std::string_view name) noexcept + : ScopedSpanGuard(SpanGuard::span(cat, prefix, name)) +{ +} + +ScopedSpanGuard +ScopedSpanGuard::freshRoot( + TraceCategory cat, + std::string_view prefix, + std::string_view name) noexcept +{ + return ScopedSpanGuard(SpanGuard::freshRoot(cat, prefix, name)); +} + +ScopedSpanGuard +ScopedSpanGuard::childSpan(std::string_view name) const noexcept +{ + return ScopedSpanGuard(impl_->guard.childSpan(name)); +} + +ScopedSpanGuard +ScopedSpanGuard::childSpan(std::string_view name, SpanContext const& parentCtx) noexcept +{ + return ScopedSpanGuard(SpanGuard::childSpan(name, parentCtx)); +} + +ScopedSpanGuard +ScopedSpanGuard::linkedSpan(std::string_view name) const noexcept +{ + return ScopedSpanGuard(impl_->guard.linkedSpan(name)); +} + +ScopedSpanGuard +ScopedSpanGuard::linkedSpan(std::string_view name, SpanContext const& linkCtx) noexcept +{ + return ScopedSpanGuard(SpanGuard::linkedSpan(name, linkCtx)); +} + +// ===== ScopedSpanGuard handoff bridge ====================================== + +ScopedSpanGuard:: +operator SpanGuard() && noexcept +{ + // The scope must be popped while its constructing context store is + // active; handing off under a different store would pop the wrong stack. + // A null guard holds no Scope, so the check is skipped. + XRPL_ASSERT( + !impl_->scope.has_value() || impl_->owner == detail::getLocalValues().get(), + "xrpl::telemetry::ScopedSpanGuard::operator SpanGuard : handoff on the " + "constructing context store"); + impl_->scope.reset(); // eager pop, on the origin store + return std::move(impl_->guard); +} + +// ===== ScopedSpanGuard context capture ===================================== + +SpanContext +ScopedSpanGuard::spanContext() const noexcept +{ + return impl_->guard.spanContext(); +} + +// ===== ScopedSpanGuard forwarding methods ================================== + +void +ScopedSpanGuard::setAttribute(std::string_view key, std::string_view value) noexcept +{ + impl_->guard.setAttribute(key, value); +} + +void +ScopedSpanGuard::setAttribute(std::string_view key, char const* value) noexcept +{ + impl_->guard.setAttribute(key, value); +} + +void +ScopedSpanGuard::setAttribute(std::string_view key, std::int64_t value) noexcept +{ + impl_->guard.setAttribute(key, value); +} + +void +ScopedSpanGuard::setAttribute(std::string_view key, double value) noexcept +{ + impl_->guard.setAttribute(key, value); +} + +void +ScopedSpanGuard::setAttribute(std::string_view key, bool value) noexcept +{ + impl_->guard.setAttribute(key, value); +} + +void +ScopedSpanGuard::setOk() noexcept +{ + impl_->guard.setOk(); +} + +void +ScopedSpanGuard::setError(std::string_view description) noexcept +{ + impl_->guard.setError(description); +} + +void +ScopedSpanGuard::addEvent(std::string_view name) noexcept +{ + impl_->guard.addEvent(name); +} + +void +ScopedSpanGuard::recordException(std::exception const& e) noexcept +{ + impl_->guard.recordException(e); +} + +void +ScopedSpanGuard::discard() noexcept +{ + // A live Scope must be popped while its constructing context store is + // active; discarding under a different store corrupts that store's + // context stack. A null guard holds no Scope, so the check is skipped. + XRPL_ASSERT( + !impl_ || !impl_->scope.has_value() || impl_->owner == detail::getLocalValues().get(), + "xrpl::telemetry::ScopedSpanGuard::discard : discarded on the " + "constructing context store"); + // Pop the scope first (under the constructing store) so no span stays + // active on the stack, then discard the span via the owned guard. + impl_->scope.reset(); + impl_->guard.discard(); +} + +ScopedSpanGuard:: +operator bool() const noexcept +{ + return impl_ && static_cast(impl_->guard); +} + +// ===== ScopedActivation ==================================================== + +struct ScopedActivation::Impl +{ + /** + * Active OTel Scope binding an externally-owned span to the current + * context store. Popped on destruction; never ends the span. + */ + otel_trace::Scope scope; + + /** + * Identity of the LocalValue store the Scope was pushed onto. The Scope + * must pop while the SAME store is active (see + * ScopedSpanGuard::ScopedImpl::owner). Initialized AFTER `scope` in the + * member init list: members initialize in declaration order, so the + * Scope's push (the first LocalValue touch on a fresh worker, which + * materializes the store) runs first, then this captures the + * now-materialized store. Capturing before the push would record + * nullptr while the destructor sees the materialized store. + */ + void const* owner; + + /** + * Push an externally-owned span onto the current context store. + * @param span The already-owned span to activate (not owned here). + */ + explicit Impl(opentelemetry::nostd::shared_ptr const& span) + : scope(span), owner(detail::getLocalValues().get()) + { + } +}; + +ScopedActivation::ScopedActivation() = default; + +ScopedActivation::ScopedActivation(std::unique_ptr impl) noexcept : impl_(std::move(impl)) +{ +} + +ScopedActivation::~ScopedActivation() +{ + // A live Scope must be popped while its constructing context store is + // active; destroying the activation under a different store corrupts + // that store's context stack. A null activation holds no Scope, so the + // check is skipped. + XRPL_ASSERT( + !impl_ || impl_->owner == detail::getLocalValues().get(), + "xrpl::telemetry::ScopedActivation::~ScopedActivation : destroyed on the " + "constructing context store"); +} + +ScopedActivation +SpanGuard::activate() const noexcept +{ + if (!impl_ || !impl_->span) + return {}; + return ScopedActivation(std::make_unique(impl_->span)); +} + +} // namespace xrpl::telemetry + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/libxrpl/telemetry/Telemetry.cpp b/src/libxrpl/telemetry/Telemetry.cpp new file mode 100644 index 00000000000..1d3eb13c7f4 --- /dev/null +++ b/src/libxrpl/telemetry/Telemetry.cpp @@ -0,0 +1,670 @@ +/** + * OpenTelemetry SDK implementation of the Telemetry interface. + * + * Compiled only when XRPL_ENABLE_TELEMETRY is defined (via CMake + * telemetry=ON). Contains: + * + * - FilteringSpanProcessor: decorator that drops spans marked with + * kDiscardedAttr before they enter the batch export queue. + * - TelemetryImpl: configures the OTel SDK with an OTLP/HTTP exporter, + * FilteringSpanProcessor wrapping a batch span processor, + * trace-ID-ratio sampler, and resource attributes. + * - NullTelemetryOtel: no-op fallback used when telemetry is compiled in + * but disabled at runtime (enabled=0 in config). + * - makeTelemetry(): factory that selects the appropriate implementation. + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::telemetry { + +namespace { + +namespace trace_api = opentelemetry::trace; +namespace trace_sdk = opentelemetry::sdk::trace; +namespace metrics_api = opentelemetry::metrics; +namespace metrics_sdk = opentelemetry::sdk::metrics; +namespace otlp_http = opentelemetry::exporter::otlp; +namespace resource = opentelemetry::sdk::resource; + +/** + * SpanProcessor decorator that drops discarded spans. + * + * Wraps a delegate processor (typically BatchSpanProcessor). In OnEnd(), + * calls DiscardScope::isActive(). If the calling thread is inside a + * DiscardScope (entered by SpanGuard::discard()), the span is silently + * dropped — never entering the batch queue, never sent over the network, + * never stored. + * + * Uses a thread-local flag rather than inspecting Recordable attributes + * because the Recordable type varies by exporter (SpanData for simple + * exporters, OtlpRecordable for OTLP) and none expose a uniform getter. + * The flag is safe because Span::End() calls OnEnd() synchronously on + * the same thread. + * + * All other methods delegate directly to the wrapped processor. + * + * Dependency diagram: + * + * +---------------------------+ + * | FilteringSpanProcessor | + * +---------------------------+ + * | - delegate_ : unique_ptr | + * | | + * +---------------------------+ + * | wraps + * +---------+-----------+ + * | BatchSpanProcessor | + * +---------------------+ + * + * @note Thread safety: OnEnd() may be called concurrently from multiple + * threads. The discard flag behind DiscardScope is thread-local, so each + * thread's discard state is independent — no synchronization needed. + */ +class FilteringSpanProcessor : public trace_sdk::SpanProcessor +{ + std::unique_ptr delegate_; + +public: + explicit FilteringSpanProcessor(std::unique_ptr delegate) + : delegate_(std::move(delegate)) + { + } + + std::unique_ptr + MakeRecordable() noexcept override + { + return delegate_->MakeRecordable(); + } + + void + OnStart( + trace_sdk::Recordable& span, + opentelemetry::trace::SpanContext const& parentContext) noexcept override + { + delegate_->OnStart(span, parentContext); + } + + void + OnEnd(std::unique_ptr&& span) noexcept override + { + if (DiscardScope::isActive()) + { + // SpanGuard::discard() is inside a DiscardScope on this thread, + // which it entered just before calling Span::End() — and End() + // invokes OnEnd() synchronously. Drop the span. + return; + } + delegate_->OnEnd(std::move(span)); + } + + bool + ForceFlush( + std::chrono::microseconds timeout = std::chrono::microseconds::max()) noexcept override + { + return delegate_->ForceFlush(timeout); + } + + bool + Shutdown(std::chrono::microseconds timeout = std::chrono::microseconds::max()) noexcept override + { + return delegate_->Shutdown(timeout); + } +}; + +/** + * No-op implementation used when XRPL_ENABLE_TELEMETRY is defined but + * setup.enabled is false at runtime. + * + * Lives in the anonymous namespace so there is no ODR conflict with the + * NullTelemetry in NullTelemetry.cpp. + */ +class NullTelemetryOtel : public Telemetry +{ + /** + * Retained configuration (unused, kept for diagnostic access). + */ + Setup const setup_; + +public: + explicit NullTelemetryOtel(Setup setup) : setup_(std::move(setup)) + { + } + + void + start() override + { + Telemetry::setInstance(this); + } + + void + stop() override + { + Telemetry::setInstance(nullptr); + } + + [[nodiscard]] bool + isEnabled() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceTransactions() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceConsensus() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceRpc() const override + { + return false; + } + + [[nodiscard]] bool + shouldTracePeer() const override + { + return false; + } + + [[nodiscard]] bool + shouldTraceLedger() const override + { + return false; + } + + [[nodiscard]] std::string const& + getConsensusTraceStrategy() const override + { + return setup_.consensusTraceStrategy; + } + + opentelemetry::nostd::shared_ptr + getTracer(std::string_view) override + { + static auto noopTracer = + opentelemetry::nostd::shared_ptr(new trace_api::NoopTracer()); + return noopTracer; + } + + opentelemetry::nostd::shared_ptr + getMeter(std::string_view name) override + { + // Serve a meter from a process-wide noop provider, mirroring the + // noop tracer above. Instruments created from it are inert. + static auto noopProvider = opentelemetry::nostd::shared_ptr( + new metrics_api::NoopMeterProvider()); + return noopProvider->GetMeter(std::string(name), std::string(kMeterVersion)); + } + + opentelemetry::nostd::shared_ptr + startSpan(std::string_view, trace_api::SpanKind) override + { + return opentelemetry::nostd::shared_ptr(new trace_api::NoopSpan(nullptr)); + } + + opentelemetry::nostd::shared_ptr + startSpan(std::string_view, opentelemetry::context::Context const&, trace_api::SpanKind) + override + { + return opentelemetry::nostd::shared_ptr(new trace_api::NoopSpan(nullptr)); + } +}; + +/** + * Full OTel SDK implementation that exports trace spans via OTLP/HTTP. + * + * Configures an OTLP/HTTP exporter, batch span processor, + * TraceIdRatioBasedSampler, and resource attributes on start(). + */ +class TelemetryImpl : public Telemetry +{ + /** + * Configuration from the [telemetry] config section. + * Non-const so setServiceInstanceId() can update the instance ID + * before start() creates the OTel resource. + */ + Setup setup_; + + /** + * Journal used for log output during start/stop. + */ + beast::Journal const journal_; + + /** + * The SDK TracerProvider that owns the export pipeline. + * + * Held as std::shared_ptr so we can call ForceFlush() on shutdown. + * Wrapped in a nostd::shared_ptr when registered as the global provider. + */ + std::shared_ptr sdkProvider_; + + /** + * The SDK MeterProvider that owns the metric export pipeline. + * + * Symmetric with sdkProvider_ on the tracing side. Held as + * std::shared_ptr so we can ForceFlush() on shutdown; wrapped in a + * nostd::shared_ptr when registered as the global meter provider. + */ + std::shared_ptr meterProvider_; + + /** + * Coroutine-aware runtime-context storage, installed globally so the OTel + * ambient context follows JobQueue coroutines. Held for the process + * lifetime because it must outlive every span (SDK requirement). + */ + opentelemetry::nostd::shared_ptr contextStorage_; + +public: + TelemetryImpl(Setup setup, beast::Journal journal) : setup_(std::move(setup)), journal_(journal) + { + // Build the metrics pipeline NOW, in the constructor, so the global + // MeterProvider is published before any subsystem is constructed. + // beast::insight instruments are created eagerly in subsystem + // constructors (e.g. LedgerMaster, NetworkOPs, ServerHandler), which + // run during ApplicationImp's member-init list — long before start(). + // opentelemetry-cpp has no proxy MeterProvider, so an instrument + // created before SetMeterProvider() binds to the noop provider forever. + // Tracing does not have this problem because getTracer() is called + // fresh at each span creation (runtime, after start()). + // + // The metrics resource uses setup_.serviceInstanceId as provided by + // config. A later setServiceInstanceId() (node-key fallback) cannot + // change this immutable resource, so operators relying on the node-key + // identity should set [telemetry] service_instance_id explicitly. + initMetrics(); + } + + void + setServiceInstanceId(std::string const& id) override + { + setup_.serviceInstanceId = id; + } + + void + start() override + { + JLOG(journal_.info()) << "Telemetry starting: endpoint=" << setup_.exporterEndpoint + << " sampling=" << setup_.samplingRatio; + + // Configure OTLP HTTP exporter + otlp_http::OtlpHttpExporterOptions exporterOpts; + exporterOpts.url = setup_.exporterEndpoint; + if (setup_.useTls) + { + exporterOpts.ssl_ca_cert_path = setup_.tlsCertPath; + // Present a client cert for mutual TLS. When both paths are + // empty the connection falls back to one-way (server) TLS. + exporterOpts.ssl_client_cert_path = setup_.tlsClientCertPath; + exporterOpts.ssl_client_key_path = setup_.tlsClientKeyPath; + } + + auto exporter = otlp_http::OtlpHttpExporterFactory::Create(exporterOpts); + + // Configure batch processor + trace_sdk::BatchSpanProcessorOptions processorOpts; + processorOpts.max_queue_size = setup_.maxQueueSize; + processorOpts.schedule_delay_millis = std::chrono::milliseconds(setup_.batchDelay); + processorOpts.max_export_batch_size = setup_.batchSize; + + auto batchProcessor = + trace_sdk::BatchSpanProcessorFactory::Create(std::move(exporter), processorOpts); + + // Wrap batch processor with filtering processor that drops spans + // marked with kDiscardedAttr (via SpanGuard::discard()). + auto processor = std::make_unique(std::move(batchProcessor)); + + // Configure resource attributes + auto resourceAttrs = resource::Resource::Create({ + {opentelemetry::semconv::service::kServiceName, setup_.serviceName}, + {opentelemetry::semconv::service::kServiceVersion, setup_.serviceVersion}, + {opentelemetry::semconv::service::kServiceInstanceId, setup_.serviceInstanceId}, + {std::string(attr::networkId), + static_cast(setup_.networkId)}, // LCOV_EXCL_LINE + {std::string(attr::networkType), setup_.networkType}, // LCOV_EXCL_LINE + }); + + // Configure sampler. Head sampling is fixed at 1.0 (sample everything); + // setup_.samplingRatio is not config-driven. Wrap the ratio sampler in a + // ParentBasedSampler so spans with a remote parent honor the upstream + // sampled flag — this keeps keep/drop decisions coherent for a single + // distributed trace spanning multiple nodes. Volume reduction is left to + // the collector's tail sampling. + auto rootSampler = + std::make_shared(setup_.samplingRatio); + auto sampler = trace_sdk::ParentBasedSamplerFactory::Create(std::move(rootSampler)); + + // Create TracerProvider with a DeterministicIdGenerator. It returns a + // deterministic trace_id when a PendingTraceId is active on the thread, + // else a random one — letting hash-derived roots (introduced on a later + // branch) become true trace roots. Dormant until such a caller exists. + sdkProvider_ = trace_sdk::TracerProviderFactory::Create( + std::move(processor), + resourceAttrs, + std::move(sampler), + std::make_unique()); + + // Install coroutine-aware context storage BEFORE any span is created + // so the OTel ambient context follows JobQueue coroutines across + // yield/resume (fixes wrong-thread scope pop; keeps log-trace + // correlation). Must precede SetTracerProvider and the first span. + // Not reset in stop(): resetting the storage while spans may still + // exist is undefined behaviour (SDK), and by stop() all spans are + // gone, so the storage is simply left installed for process lifetime. + contextStorage_ = + opentelemetry::nostd::shared_ptr( + new CoroAwareContextStorage()); + opentelemetry::context::RuntimeContext::SetRuntimeContextStorage(contextStorage_); + + // Set as global provider + trace_api::Provider::SetTracerProvider( + opentelemetry::nostd::shared_ptr(sdkProvider_)); + + // The metrics pipeline (meterProvider_) was already built and published + // in the constructor via initMetrics(), before any subsystem could + // create a beast::insight instrument. See initMetrics(). + + // Register as the global Telemetry instance so SpanGuard factory + // methods can access it without callers passing a reference. + Telemetry::setInstance(this); + + JLOG(journal_.info()) << "Telemetry started successfully"; + } + + /** + * Build and publish the metrics pipeline (MeterProvider + periodic + * reader + OTLP exporter + histogram view). + * + * Called from the constructor, NOT start(), so the global MeterProvider + * exists before subsystems construct their beast::insight instruments + * during ApplicationImp's member-init list. The metrics resource uses + * setup_.serviceInstanceId from config; it is immutable once the provider + * is built, so a later node-key setServiceInstanceId() does not affect it. + */ + void + initMetrics() + { + // Derive the metrics endpoint from the trace endpoint by swapping + // the trailing "/v1/traces" path for "/v1/metrics". Any other URL + // shape is used as-is. + std::string metricsEndpoint = setup_.exporterEndpoint; + constexpr std::string_view tracesPath{"/v1/traces"}; + if (metricsEndpoint.ends_with(tracesPath)) + { + metricsEndpoint.replace( + metricsEndpoint.size() - tracesPath.size(), tracesPath.size(), "/v1/metrics"); + } + + // Configure OTLP HTTP metric exporter, honoring the same TLS + // options as the trace exporter. + otlp_http::OtlpHttpMetricExporterOptions metricExporterOpts; + metricExporterOpts.url = metricsEndpoint; + if (setup_.useTls) + { + metricExporterOpts.ssl_ca_cert_path = setup_.tlsCertPath; + metricExporterOpts.ssl_client_cert_path = setup_.tlsClientCertPath; + metricExporterOpts.ssl_client_key_path = setup_.tlsClientKeyPath; + } + + auto metricExporter = otlp_http::OtlpHttpMetricExporterFactory::Create(metricExporterOpts); + + // Configure periodic metric reader (1-second export interval, + // matching the beast OTelCollector path). + metrics_sdk::PeriodicExportingMetricReaderOptions readerOpts; + readerOpts.export_interval_millis = std::chrono::milliseconds(1000); + readerOpts.export_timeout_millis = std::chrono::milliseconds(500); + + auto reader = metrics_sdk::PeriodicExportingMetricReaderFactory::Create( + std::move(metricExporter), readerOpts); + + // Metrics resource: same attributes as the tracer resource so metrics + // and traces share one identity. Built here (not shared with start()) + // because start() runs later; serviceInstanceId comes from config. + auto resourceAttrs = resource::Resource::Create({ + {opentelemetry::semconv::service::kServiceName, setup_.serviceName}, + {opentelemetry::semconv::service::kServiceVersion, setup_.serviceVersion}, + {opentelemetry::semconv::service::kServiceInstanceId, setup_.serviceInstanceId}, + {std::string(attr::networkId), static_cast(setup_.networkId)}, + {std::string(attr::networkType), setup_.networkType}, + }); + + // Create MeterProvider with the shared resource, then attach reader. + meterProvider_ = metrics_sdk::MeterProviderFactory::Create( + std::make_unique(), resourceAttrs); + meterProvider_->AddMetricReader(std::move(reader)); + + // Histogram view: SpanMetrics-compatible bucket boundaries (ms) so + // histogram instruments align with the collector's SpanMetrics. The + // view is created with an EMPTY name so it applies the buckets WITHOUT + // renaming instruments — a non-empty view name would collapse every + // matching histogram (ios_latency, rpc_size, rpc_time, pathfind_*) + // into a single series under that one name. + auto histogramSelector = metrics_sdk::InstrumentSelectorFactory::Create( + metrics_sdk::InstrumentType::kHistogram, "*", "ms"); + // Meter selector MUST match the meter name used by getMeter() and the + // beast OTelCollector (kMeterName = "xrpld"); otherwise this histogram + // view never applies and duration histograms fall back to the SDK + // default boundaries instead of these SpanMetrics-aligned buckets. + auto meterSelector = + metrics_sdk::MeterSelectorFactory::Create(std::string(kMeterName), "", ""); + auto histogramConfig = std::make_shared(); + histogramConfig->boundaries_ = + std::vector{1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 5000.0}; + auto histogramView = metrics_sdk::ViewFactory::Create( + "", // empty name: keep each instrument's own name, only set buckets + "SpanMetrics-compatible histogram buckets", + metrics_sdk::AggregationType::kHistogram, + histogramConfig); + + meterProvider_->AddView( + std::move(histogramSelector), std::move(meterSelector), std::move(histogramView)); + + // Publish as the global meter provider so developers (and the beast + // OTelCollector shim) reach the same pipeline. + metrics_api::Provider::SetMeterProvider( + opentelemetry::nostd::shared_ptr(meterProvider_)); + } + + void + stop() override + { + JLOG(journal_.info()) << "Telemetry stopping"; + + // Unregister global instance before tearing down the pipeline. + Telemetry::setInstance(nullptr); + + if (sdkProvider_) + { + // Force flush with timeout to avoid blocking indefinitely + // when the OTLP endpoint is unreachable. + sdkProvider_->ForceFlush(std::chrono::milliseconds(5000)); + // TODO: sdkProvider_ is not thread-safe. This reset() races with + // getTracer() if any thread is still calling startSpan(). + // Currently safe because Application::stop() shuts down + // serverHandler_, overlay_, and jobQueue_ before calling + // telemetry_->stop() — so no callers should remain. If the + // shutdown order ever changes, add an std::atomic stopped_ + // flag checked in getTracer() to make this robust. + sdkProvider_.reset(); + trace_api::Provider::SetTracerProvider( + opentelemetry::nostd::shared_ptr( + new trace_api::NoopTracerProvider())); + } + + if (meterProvider_) + { + // Mirror the tracer teardown: bounded flush, drop our provider, + // then restore a noop global provider. Same shutdown-order + // caveat as sdkProvider_ above applies to getMeter() callers. + meterProvider_->ForceFlush(std::chrono::milliseconds(5000)); + meterProvider_.reset(); + metrics_api::Provider::SetMeterProvider( + opentelemetry::nostd::shared_ptr( + new metrics_api::NoopMeterProvider())); + } + JLOG(journal_.info()) << "Telemetry stopped"; + } + + [[nodiscard]] bool + isEnabled() const override + { + return true; + } + + [[nodiscard]] bool + shouldTraceTransactions() const override + { + return setup_.traceTransactions; + } + + [[nodiscard]] bool + shouldTraceConsensus() const override + { + return setup_.traceConsensus; + } + + [[nodiscard]] bool + shouldTraceRpc() const override + { + return setup_.traceRpc; + } + + [[nodiscard]] bool + shouldTracePeer() const override + { + return setup_.tracePeer; + } + + [[nodiscard]] bool + shouldTraceLedger() const override + { + return setup_.traceLedger; + } + + [[nodiscard]] std::string const& + getConsensusTraceStrategy() const override + { + return setup_.consensusTraceStrategy; + } + + opentelemetry::nostd::shared_ptr + getTracer(std::string_view name = kTracerName) override + { + if (!sdkProvider_) + { + return trace_api::Provider::GetTracerProvider()->GetTracer(std::string(name)); + } + return sdkProvider_->GetTracer(std::string(name)); + } + + opentelemetry::nostd::shared_ptr + getMeter(std::string_view name = kMeterName) override + { + if (!meterProvider_) + { + return metrics_api::Provider::GetMeterProvider()->GetMeter( + std::string(name), std::string(kMeterVersion)); + } + return meterProvider_->GetMeter(std::string(name), std::string(kMeterVersion)); + } + + opentelemetry::nostd::shared_ptr + startSpan(std::string_view name, trace_api::SpanKind kind) override + { + auto tracer = getTracer(); + trace_api::StartSpanOptions opts; + opts.kind = kind; + return tracer->StartSpan(std::string(name), opts); + } + + opentelemetry::nostd::shared_ptr + startSpan( + std::string_view name, + opentelemetry::context::Context const& parentContext, + trace_api::SpanKind kind) override + { + auto tracer = getTracer(); + trace_api::StartSpanOptions opts; + opts.kind = kind; + opts.parent = parentContext; + return tracer->StartSpan(std::string(name), opts); + } +}; + +} // namespace + +std::unique_ptr +makeTelemetry(Telemetry::Setup const& setup, beast::Journal journal) +{ + if (setup.enabled) + { + return std::make_unique(setup, journal); + } + return std::make_unique(setup); +} + +} // namespace xrpl::telemetry + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/libxrpl/telemetry/TelemetryConfig.cpp b/src/libxrpl/telemetry/TelemetryConfig.cpp new file mode 100644 index 00000000000..d122393e51a --- /dev/null +++ b/src/libxrpl/telemetry/TelemetryConfig.cpp @@ -0,0 +1,161 @@ +/** + * Parser for the [telemetry] section of xrpld.cfg. + * + * Reads configuration values from the config file and populates a + * Telemetry::Setup struct. All options have sensible defaults so the + * section can be minimal or omitted entirely. + * + * See cfg/xrpld-example.cfg for the full list of available options. + */ + +#include +#include +#include + +#include +#include +#include +#include + +namespace xrpl::telemetry { + +namespace { + +/** + * Config key names for the [telemetry] section. + * + * Each must match the corresponding option documented in + * cfg/xrpld-example.cfg verbatim. Defined as `char const*` so they + * pass to Section::valueOr() (which takes `std::string const&`) + * without an explicit conversion, exactly as a literal would. + */ +namespace key { +constexpr char const* enabled = "enabled"; +constexpr char const* serviceName = "service_name"; +constexpr char const* serviceInstanceId = "service_instance_id"; +constexpr char const* endpoint = "endpoint"; +constexpr char const* useTls = "use_tls"; +constexpr char const* tlsCaCert = "tls_ca_cert"; +constexpr char const* tlsClientCert = "tls_client_cert"; +constexpr char const* tlsClientKey = "tls_client_key"; +constexpr char const* batchSize = "batch_size"; +constexpr char const* batchDelayMs = "batch_delay_ms"; +constexpr char const* maxQueueSize = "max_queue_size"; +constexpr char const* traceTransactions = "trace_transactions"; +constexpr char const* traceConsensus = "trace_consensus"; +constexpr char const* traceRpc = "trace_rpc"; +constexpr char const* tracePeer = "trace_peer"; +constexpr char const* traceLedger = "trace_ledger"; +} // namespace key + +/** + * Default values applied when a key is absent from the config. + * + * @note serviceName mirrors SystemParameters' systemName() ("xrpld") but + * is duplicated here as a literal: the telemetry module deliberately does + * not link xrpl.libxrpl.protocol, so including SystemParameters.h would + * introduce an undeclared cross-module dependency. + */ +namespace dflt { +constexpr char const* serviceName = "xrpld"; +constexpr char const* endpoint = "http://localhost:4318/v1/traces"; +constexpr std::uint32_t batchSize = 512u; +constexpr std::uint32_t batchDelayMs = 5000u; +constexpr std::uint32_t maxQueueSize = 2048u; +} // namespace dflt + +} // namespace + +/** + * Derive a human-readable network type label from the numeric network ID. + * + * Declared in Telemetry.h; shared by the trace and metric export paths so + * both stamp the same xrpl.network.type resource attribute value. + * + * @param networkId The network identifier from [network_id] config. + * @return "mainnet", "testnet", "devnet", or "unknown" for other values. + */ +std::string +networkTypeFromId(std::uint32_t networkId) +{ + switch (networkId) + { + case 0: + return "mainnet"; + case 1: + return "testnet"; + case 2: + return "devnet"; + default: + return "unknown"; + } +} + +Telemetry::Setup +makeTelemetrySetup( + Section const& section, + std::string const& nodePublicKey, + std::string const& version, + std::uint32_t networkId) +{ + Telemetry::Setup setup; + + setup.enabled = section.valueOr(key::enabled, 0) != 0; + setup.serviceName = section.valueOr(key::serviceName, dflt::serviceName); + setup.serviceVersion = version; + setup.serviceInstanceId = section.valueOr(key::serviceInstanceId, nodePublicKey); + + setup.exporterEndpoint = section.valueOr(key::endpoint, dflt::endpoint); + + setup.useTls = section.valueOr(key::useTls, 0) != 0; + setup.tlsCertPath = section.valueOr(key::tlsCaCert, ""); + setup.tlsClientCertPath = section.valueOr(key::tlsClientCert, ""); + setup.tlsClientKeyPath = section.valueOr(key::tlsClientKey, ""); + + // Mutual TLS needs both the client certificate and its private key. + // Supplying only one fails later with a cryptic SSL handshake error, so + // reject the partial configuration here with an actionable message. + if (setup.tlsClientCertPath.empty() != setup.tlsClientKeyPath.empty()) + { + Throw( + "[telemetry] tls_client_cert and tls_client_key must be set together " + "(set both for mutual TLS, or neither for one-way TLS)."); + } + + // Mutual TLS only takes effect when TLS is on. Certificate paths set with + // use_tls=0 would be silently ignored and the exporter would connect in + // plaintext, so reject that contradiction instead of failing open. + if (!setup.tlsClientCertPath.empty() && !setup.useTls) + { + Throw( + "[telemetry] tls_client_cert/tls_client_key require use_tls=1 " + "(set use_tls=1 to enable mutual TLS, or remove the cert paths)."); + } + + // Head sampling is intentionally fixed at 1.0 (sample everything) and is + // not read from config. A per-node ratio would let nodes make divergent + // keep/drop decisions for the same distributed trace, producing broken + // traces; volume reduction is delegated to the collector's tail sampling. + // setup.samplingRatio is a const member fixed at 1.0; nothing to parse. + + setup.batchSize = section.valueOr(key::batchSize, dflt::batchSize); + setup.batchDelay = std::chrono::milliseconds{ + section.valueOr(key::batchDelayMs, dflt::batchDelayMs)}; + setup.maxQueueSize = section.valueOr(key::maxQueueSize, dflt::maxQueueSize); + + setup.networkId = networkId; + setup.networkType = networkTypeFromId(networkId); + + setup.traceTransactions = section.valueOr(key::traceTransactions, 1) != 0; + setup.traceConsensus = section.valueOr(key::traceConsensus, 1) != 0; + setup.traceRpc = section.valueOr(key::traceRpc, 1) != 0; + setup.tracePeer = section.valueOr(key::tracePeer, 1) != 0; + setup.traceLedger = section.valueOr(key::traceLedger, 1) != 0; + + setup.consensusTraceStrategy = + section.valueOr("consensus_trace_strategy", "deterministic"); + + return setup; +} + +} // namespace xrpl::telemetry diff --git a/src/libxrpl/tx/Transactor.cpp b/src/libxrpl/tx/Transactor.cpp index 4b562692d75..f541d788cf9 100644 --- a/src/libxrpl/tx/Transactor.cpp +++ b/src/libxrpl/tx/Transactor.cpp @@ -34,13 +34,16 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include +#include #include #include @@ -1591,6 +1594,27 @@ Transactor::checkInvariants(TER result, XRPAmount fee) ApplyResult Transactor::operator()() { + // Derive the trace_id from the transaction id so this apply-stage span + // shares one trace with the preflight and preclaim spans (which also use + // hashSpan on the same id). + auto const txID = ctx_.tx.getTransactionID(); + auto span = telemetry::SpanGuard::hashSpan( + telemetry::TraceCategory::Transactions, + telemetry::tx_apply_span::transactor, + txID.data(), + txID.kBytes); + // "apply" — the third apply-pipeline stage, after preflight and preclaim. + span.setAttribute(telemetry::tx_apply_span::attr::stage, telemetry::tx_apply_span::val::apply); + if (auto const* fmt = TxFormats::getInstance().findByType(ctx_.tx.getTxnType())) + span.setAttribute(telemetry::tx_apply_span::attr::txType, fmt->getName().c_str()); + // The ledger being worked on (seq + parent hash) — correlates this apply + // stage to the ledger/consensus trace it is building into. + span.setAttribute( + telemetry::tx_apply_span::attr::currentLedgerSeq, static_cast(view().seq())); + span.setAttribute( + telemetry::tx_apply_span::attr::currentLedgerHash, + to_string(view().header().parentHash).c_str()); + JLOG(j_.trace()) << "apply: " << ctx_.tx.getTransactionID(); // These global updates really should have been for every Transaction @@ -1714,6 +1738,14 @@ Transactor::operator()() JLOG(j_.trace()) << (applied ? "applied " : "not applied ") << transToken(result); + span.setAttribute(telemetry::tx_apply_span::attr::terResult, transToken(result).c_str()); + span.setAttribute(telemetry::tx_apply_span::attr::applied, applied); + // Mark the span as errored when the transaction was not applied or the + // engine result is not a success, so failed applies surface in span-status + // error counts alongside preflight and preclaim. + if (!applied || !isTesSuccess(result)) + span.setError(transToken(result)); + return {result, applied, metadata}; } diff --git a/src/libxrpl/tx/applySteps.cpp b/src/libxrpl/tx/applySteps.cpp index 5af4f621a79..f2f99e95364 100644 --- a/src/libxrpl/tx/applySteps.cpp +++ b/src/libxrpl/tx/applySteps.cpp @@ -12,13 +12,16 @@ #include #include #include +#include #include #include +#include #include #include #include #include +#include #include #pragma push_macro("TRANSACTION") #undef TRANSACTION @@ -50,6 +53,71 @@ struct UnknownTxnType : std::exception } }; +/** + * Look up the human-readable transaction type name for span attributes. + * Returns nullptr if the type is unknown so the caller can skip the + * attribute rather than emit an empty value. + */ +char const* +txTypeName(TxType txnType) +{ + if (auto const* fmt = TxFormats::getInstance().findByType(txnType)) + return fmt->getName().c_str(); + return nullptr; +} + +/** + * Create a deterministic-trace span for an apply-pipeline stage. + * + * The trace_id is derived from txID[0:16] so the preflight, preclaim, and + * transactor spans of one transaction share a trace even though they run + * sequentially and often on different threads. Sets the stage, tx_type, and + * (after the stage runs) ter_result attributes that drive the collector + * spanmetrics dimensions. A no-op when telemetry is disabled. + * + * @param name Full span name (tx_apply_span::preflight / ::preclaim). + * @param stage Stage attribute value (tx_apply_span::val::*). + * @param tx The transaction supplying the id and type. + * @param curLedgerSeq Seq of the ledger being worked on, set as the + * current_ledger_seq attribute. Passed by the + * view-bearing stages (preclaim); nullopt for the + * stateless preflight, which then omits it. + * @param curLedgerParentHash Parent hash of that ledger, set as + * current_ledger_hash. nullptr to omit. + */ +[[nodiscard]] telemetry::SpanGuard +makeStageSpan( + std::string_view name, + std::string_view stage, + STTx const& tx, + std::optional curLedgerSeq = std::nullopt, + uint256 const* curLedgerParentHash = nullptr) +{ + auto const txID = tx.getTransactionID(); + auto span = telemetry::SpanGuard::hashSpan( + telemetry::TraceCategory::Transactions, name, txID.data(), txID.kBytes); + // Guard the type lookup behind the active check: preflight runs for every + // transaction, so findByType() must not run when tracing is off/disabled. + if (span) + { + span.setAttribute(telemetry::tx_apply_span::attr::stage, stage); + if (char const* typeName = txTypeName(tx.getTxnType())) + span.setAttribute(telemetry::tx_apply_span::attr::txType, typeName); + // The ledger being worked on — set only by the view-bearing stages + // (preclaim/transactor). Preflight is stateless and passes nullopt, so + // it carries no ledger attribute (documented exception). + if (curLedgerSeq) + span.setAttribute( + telemetry::tx_apply_span::attr::currentLedgerSeq, + static_cast(*curLedgerSeq)); + if (curLedgerParentHash) + span.setAttribute( + telemetry::tx_apply_span::attr::currentLedgerHash, + to_string(*curLedgerParentHash).c_str()); + } + return span; +} + // Call a lambda with the concrete transaction type as a template parameter // throw an "UnknownTxnType" exception on error template @@ -131,86 +199,136 @@ consequencesHelper(PreflightContext const& ctx) static std::pair invokePreflight(PreflightContext const& ctx) { + // Trace the preflight stage. The span shares the transaction's + // deterministic trace_id so it correlates with preclaim and transactor. + auto span = makeStageSpan( + telemetry::tx_apply_span::preflight, telemetry::tx_apply_span::val::preflight, ctx.tx); try { - return withTxnType(ctx.rules, ctx.tx.getTxnType(), [&]() { + auto result = withTxnType(ctx.rules, ctx.tx.getTxnType(), [&]() { auto const tec = Transactor::invokePreflight(ctx); return std::make_pair( tec, isTesSuccess(tec) ? consequencesHelper(ctx) : TxConsequences{tec}); }); + if (span) + { + span.setAttribute( + telemetry::tx_apply_span::attr::terResult, transToken(result.first).c_str()); + // Mark the span as errored when preflight rejects the transaction so + // failed stages surface in span-status error counts. + if (!isTesSuccess(result.first)) + span.setError(transToken(result.first)); + } + return result; } catch (UnknownTxnType const& e) { // Should never happen // LCOV_EXCL_START JLOG(ctx.j.fatal()) << "Unknown transaction type in preflight: " << e.txnType; + span.recordException(e); UNREACHABLE("xrpl::invokePreflight : unknown transaction type"); return {temUNKNOWN, TxConsequences{temUNKNOWN}}; // LCOV_EXCL_STOP } + catch (std::exception const& e) + { + // The caller's preflight() maps this to tefEXCEPTION. Record it on the + // span before unwinding so per-stage error counts include exceptions. + span.setAttribute( + telemetry::tx_apply_span::attr::terResult, transToken(tefEXCEPTION).c_str()); + span.recordException(e); + throw; + } } static TER invokePreclaim(PreclaimContext const& ctx) { + // Trace the preclaim stage under the transaction's deterministic trace_id. + // Preclaim has a ledger view, so tag the span with the ledger being worked + // on (seq + parent hash) to correlate it to the ledger/consensus trace. + auto span = makeStageSpan( + telemetry::tx_apply_span::preclaim, + telemetry::tx_apply_span::val::preclaim, + ctx.tx, + ctx.view.seq(), + &ctx.view.header().parentHash); try { // use name hiding to accomplish compile-time polymorphism of static // class functions for Transactor and derived classes. - return withTxnType(ctx.view.rules(), ctx.tx.getTxnType(), [&]() -> TER { - // preclaim functionality is divided into two sections: - // 1. Up to and including the signature check: returns NotTEC. - // All transaction checks before and including checkSign - // MUST return NotTEC, or something more restrictive. - // Allowing tec results in these steps risks theft or - // destruction of funds, as a fee will be charged before the - // signature is checked. - // 2. After the signature check: returns TER. - - // If the transactor requires a valid account and the - // transaction doesn't list one, preflight will have already - // a flagged a failure. - auto const id = ctx.tx.getAccountID(sfAccount); - - if (id != beast::kZero) - { - if (NotTEC const preSigResult = [&]() -> NotTEC { - if (NotTEC const result = T::checkSeqProxy(ctx.view, ctx.tx, ctx.j)) - return result; - - if (NotTEC const result = T::checkPriorTxAndLastLedger(ctx)) - return result; - - if (NotTEC const result = T::checkSponsor(ctx.view, ctx.tx)) - return result; - - if (NotTEC const result = - Transactor::invokeCheckPermission(ctx.view, ctx.tx)) - return result; - - if (NotTEC const result = T::checkSign(ctx)) - return result; - - return tesSUCCESS; - }()) - return preSigResult; - - if (TER const result = T::checkFee(ctx, calculateBaseFee(ctx.view, ctx.tx))) - return result; - } - - return T::preclaim(ctx); - }); + TER const preclaimTer = + withTxnType(ctx.view.rules(), ctx.tx.getTxnType(), [&]() -> TER { + // preclaim functionality is divided into two sections: + // 1. Up to and including the signature check: returns NotTEC. + // All transaction checks before and including checkSign + // MUST return NotTEC, or something more restrictive. + // Allowing tec results in these steps risks theft or + // destruction of funds, as a fee will be charged before the + // signature is checked. + // 2. After the signature check: returns TER. + + // If the transactor requires a valid account and the + // transaction doesn't list one, preflight will have already + // a flagged a failure. + auto const id = ctx.tx.getAccountID(sfAccount); + + if (id != beast::kZero) + { + if (NotTEC const preSigResult = [&]() -> NotTEC { + if (NotTEC const result = T::checkSeqProxy(ctx.view, ctx.tx, ctx.j)) + return result; + + if (NotTEC const result = T::checkPriorTxAndLastLedger(ctx)) + return result; + + if (NotTEC const result = T::checkSponsor(ctx.view, ctx.tx)) + return result; + + if (NotTEC const result = + Transactor::invokeCheckPermission(ctx.view, ctx.tx)) + return result; + + if (NotTEC const result = T::checkSign(ctx)) + return result; + + return tesSUCCESS; + }()) + return preSigResult; + + if (TER const result = T::checkFee(ctx, calculateBaseFee(ctx.view, ctx.tx))) + return result; + } + + return T::preclaim(ctx); + }); + if (span) + { + span.setAttribute( + telemetry::tx_apply_span::attr::terResult, transToken(preclaimTer).c_str()); + } + return preclaimTer; } catch (UnknownTxnType const& e) { // Should never happen // LCOV_EXCL_START JLOG(ctx.j.fatal()) << "Unknown transaction type in preclaim: " << e.txnType; + span.recordException(e); UNREACHABLE("xrpl::invokePreclaim : unknown transaction type"); return temUNKNOWN; // LCOV_EXCL_STOP } + catch (std::exception const& e) + { + // The caller's preclaim() maps this to tefEXCEPTION. Record it on the + // span before unwinding so per-stage error counts include exceptions. + span.setAttribute( + telemetry::tx_apply_span::attr::terResult, transToken(tefEXCEPTION).c_str()); + span.recordException(e); + throw; + } } /** diff --git a/src/test/basics/PerfLog_test.cpp b/src/test/basics/PerfLog_test.cpp index 41b5f81f5d5..792934f0fef 100644 --- a/src/test/basics/PerfLog_test.cpp +++ b/src/test/basics/PerfLog_test.cpp @@ -533,7 +533,7 @@ class PerfLog_test : public beast::unit_test::Suite // the jobs data with every addition. for (int i = 0; i < jobs.size(); ++i) { - perfLog->jobQueue(jobs[i].type); + perfLog->jobQueue(jobs[i].type, jobs[i].typeName); json::Value const jqCounters{perfLog->countersJson()[jss::job_queue]}; BEAST_EXPECT(jqCounters.size() == i + 2); @@ -581,7 +581,8 @@ class PerfLog_test : public beast::unit_test::Suite // be half as many queued as started... for (int i = 0; i < jobs.size(); ++i) { - perfLog->jobStart(jobs[i].type, microseconds{i + 1}, steady_clock::now(), i * 2); + perfLog->jobStart( + jobs[i].type, jobs[i].typeName, microseconds{i + 1}, steady_clock::now(), i * 2); std::this_thread::sleep_for(microseconds(10)); // Check each jobType counter entry. @@ -623,7 +624,8 @@ class PerfLog_test : public beast::unit_test::Suite BEAST_EXPECT(total[jss::running_duration_us] == "0"); } - perfLog->jobStart(jobs[i].type, microseconds{0}, steady_clock::now(), (i * 2) + 1); + perfLog->jobStart( + jobs[i].type, jobs[i].typeName, microseconds{0}, steady_clock::now(), (i * 2) + 1); std::this_thread::sleep_for(microseconds{10}); // Verify that every entry in jobs appears twice in currents. @@ -651,7 +653,7 @@ class PerfLog_test : public beast::unit_test::Suite // A number of the computations in this loop care about the // number of jobs that have finished. Make that available. int const finished = ((jobs.size() - i) * 2) - 1; - perfLog->jobFinish(jobs[i].type, microseconds(finished), (i * 2) + 1); + perfLog->jobFinish(jobs[i].type, jobs[i].typeName, microseconds(finished), (i * 2) + 1); std::this_thread::sleep_for(microseconds(10)); json::Value const jqCounters{perfLog->countersJson()[jss::job_queue]}; @@ -697,7 +699,7 @@ class PerfLog_test : public beast::unit_test::Suite BEAST_EXPECT(jsonToUInt64(total[jss::running_duration_us]) == runningDur); } - perfLog->jobFinish(jobs[i].type, microseconds(finished + 1), (i * 2)); + perfLog->jobFinish(jobs[i].type, jobs[i].typeName, microseconds(finished + 1), (i * 2)); std::this_thread::sleep_for(microseconds(10)); // Verify that the two jobs we just finished no longer appear in @@ -891,25 +893,25 @@ class PerfLog_test : public beast::unit_test::Suite }; // Start an ID that's too large. - perfLog->jobStart(jobType, microseconds{11}, steady_clock::now(), 2); + perfLog->jobStart(jobType, jobTypeName, microseconds{11}, steady_clock::now(), 2); std::this_thread::sleep_for(microseconds{10}); verifyCounters(perfLog->countersJson(), 1, 0, 11, 0); verifyEmptyCurrent(perfLog->currentJson()); // Start a negative ID - perfLog->jobStart(jobType, microseconds{13}, steady_clock::now(), -1); + perfLog->jobStart(jobType, jobTypeName, microseconds{13}, steady_clock::now(), -1); std::this_thread::sleep_for(microseconds{10}); verifyCounters(perfLog->countersJson(), 2, 0, 24, 0); verifyEmptyCurrent(perfLog->currentJson()); // Finish the too large ID - perfLog->jobFinish(jobType, microseconds{17}, 2); + perfLog->jobFinish(jobType, jobTypeName, microseconds{17}, 2); std::this_thread::sleep_for(microseconds{10}); verifyCounters(perfLog->countersJson(), 2, 1, 24, 17); verifyEmptyCurrent(perfLog->currentJson()); // Finish the negative ID - perfLog->jobFinish(jobType, microseconds{19}, -1); + perfLog->jobFinish(jobType, jobTypeName, microseconds{19}, -1); std::this_thread::sleep_for(microseconds{10}); verifyCounters(perfLog->countersJson(), 2, 2, 24, 36); verifyEmptyCurrent(perfLog->currentJson()); diff --git a/src/test/core/JobQueue_test.cpp b/src/test/core/JobQueue_test.cpp index aa6a0d60a8e..2f262282dca 100644 --- a/src/test/core/JobQueue_test.cpp +++ b/src/test/core/JobQueue_test.cpp @@ -1,14 +1,307 @@ #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include #include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include namespace xrpl::test { +namespace { + +/** + * A beast::insight::Collector that remembers the last value set on every + * gauge it created, keyed by gauge name. + * + * Needed because the production collectors are write-only sinks: StatsD + * sends over UDP and NullCollector discards. `JobQueue::collect()` assigns + * to the per-job-type gauges, so the only way to assert on the published + * values in-process is to supply a Collector that keeps them. + * + * RecordingCollector (Collector) + * | makeGauge(name) + * v + * RecordingGaugeImpl (GaugeImpl) --writes--> Values (shared, mutex-guarded) + * ^ ^ + * | | + * JobQueue::collect() assigns gaugeValue(name) reads + * + * Only gauges are recorded; counters, events, meters and hooks get inert + * implementations, because no assertion here needs them. The hook handler is + * kept so the test can invoke the collection pass on demand rather than + * waiting for a collector's own timer -- that makes the reads deterministic. + * + * @note Thread-safe: the value map is guarded by its own mutex, because + * `JobQueue::collect()` may run on a different thread from the assertions. + */ +class RecordingCollector : public beast::insight::Collector +{ +public: + /** + * Shared storage so gauge handles outliving the collector stay valid. + */ + struct Values + { + std::mutex mutex; + std::map gauges; + }; + +private: + /** + * A gauge that writes each assigned value into the shared map. + */ + class RecordingGaugeImpl : public beast::insight::GaugeImpl + { + std::shared_ptr values_; + std::string name_; + + public: + RecordingGaugeImpl(std::shared_ptr values, std::string name) + : values_(std::move(values)), name_(std::move(name)) + { + } + + void + set(value_type value) override + { + std::scoped_lock const lock(values_->mutex); + values_->gauges[name_] = value; + } + + void + increment(difference_type amount) override + { + std::scoped_lock const lock(values_->mutex); + values_->gauges[name_] += static_cast(amount); + } + }; + + /** @{ */ + /** + * Inert implementations for the metric kinds no assertion reads. + */ + class InertCounterImpl : public beast::insight::CounterImpl + { + void + increment(value_type) override + { + } + }; + + class InertEventImpl : public beast::insight::EventImpl + { + void + notify(value_type const&) override + { + } + }; + + class InertMeterImpl : public beast::insight::MeterImpl + { + void + increment(value_type) override + { + } + }; + + /** + * A hook that owns its handler, so releasing the Hook releases the + * handler with it. + * + * The collector holds only a weak reference (see `hooks_`). That + * reproduces the documented beast::insight lifetime rule -- "when the + * last reference goes away, the metric is no longer collected" -- and + * matters here because `JobQueue::~JobQueue()` unhooks by assigning an + * empty Hook. A collector holding the handler strongly would keep + * calling back into a destroyed JobQueue. + */ + class InertHookImpl : public beast::insight::HookImpl + { + public: + explicit InertHookImpl(HandlerType handler) : handler_(std::move(handler)) + { + } + + void + invoke() const + { + if (handler_) + handler_(); + } + + private: + HandlerType handler_; + }; + /** @} */ + + std::shared_ptr values_{std::make_shared()}; + std::vector> hooks_; + +public: + beast::insight::Hook + makeHook(beast::insight::HookImpl::HandlerType const& handler) override + { + auto impl = std::make_shared(handler); + hooks_.push_back(impl); + return beast::insight::Hook(std::move(impl)); + } + + beast::insight::Counter + makeCounter(std::string const&) override + { + return beast::insight::Counter(std::make_shared()); + } + + beast::insight::Event + makeEvent(std::string const&) override + { + return beast::insight::Event(std::make_shared()); + } + + beast::insight::Gauge + makeGauge(std::string const& name) override + { + return beast::insight::Gauge(std::make_shared(values_, name)); + } + + beast::insight::Meter + makeMeter(std::string const&) override + { + return beast::insight::Meter(std::make_shared()); + } + + /** + * Run every still-live hook, i.e. force one collection pass. + * + * Expired hooks are skipped rather than resurrected, so calling this + * after the JobQueue has been destroyed is a no-op instead of a + * use-after-free. + */ + void + runHooks() const + { + for (auto const& weak : hooks_) + { + if (auto const hook = weak.lock()) + hook->invoke(); + } + } + + /** + * The last value published for @p name. + * + * @return The value, or std::nullopt when no gauge of that name has + * ever been written -- which distinguishes "gauge absent" from + * "gauge present and reading zero". + */ + [[nodiscard]] std::optional + gaugeValue(std::string const& name) const + { + std::scoped_lock const lock(values_->mutex); + auto const iter = values_->gauges.find(name); + if (iter == values_->gauges.end()) + return std::nullopt; + return iter->second; + } +}; + +/** + * A perf::PerfLog that ignores everything. + * + * JobQueue requires a PerfLog reference; these tests assert on gauges, not + * on the perf hooks, so every override is empty. + */ +class SilentPerfLog : public perf::PerfLog +{ + void + rpcStart(std::string const&, std::uint64_t) override + { + } + void + rpcFinish(std::string const&, std::uint64_t) override + { + } + void + rpcError(std::string const&, std::uint64_t) override + { + } + void + jobQueue(JobType, std::string const&) override + { + } + void + jobStart( + JobType, + std::string const&, + std::chrono::microseconds, + std::chrono::time_point, + int) override + { + } + void + jobFinish(JobType, std::string const&, std::chrono::microseconds, int) override + { + } + [[nodiscard]] json::Value + countersJson() const override + { + return json::Value(); + } + [[nodiscard]] json::Value + currentJson() const override + { + return json::Value(); + } + void + resizeJobs(int) override + { + } + void + rotate() override + { + } +}; + +// Gauge-name suffixes. Aliased from JobTypeData rather than re-spelled, so a +// rename there fails here instead of silently asserting on a stale name. +/** @{ */ +constexpr auto& kSuffixWaiting = JobTypeData::kSuffixWaiting; +constexpr auto& kSuffixRunning = JobTypeData::kSuffixRunning; +constexpr auto& kSuffixDeferred = JobTypeData::kSuffixDeferred; +/** @} */ + +} // namespace + //------------------------------------------------------------------------------ class JobQueue_test : public beast::unit_test::Suite @@ -130,12 +423,298 @@ class JobQueue_test : public beast::unit_test::Suite } } + //-------------------------------------------------------------------------- + // Per-job-type saturation gauges (waiting / running / deferred) + //-------------------------------------------------------------------------- + + /** + * Owns a JobQueue wired to a RecordingCollector. + * + * A standalone JobQueue rather than `env.app().getJobQueue()`, for two + * reasons: the application's queue uses a write-only collector whose + * gauge values cannot be read back, and it carries background jobs whose + * timing would make exact counts unreproducible. Constructed with the + * given thread count so a concurrency limit can be exceeded on demand. + */ + struct GaugeFixture + { + Logs logs{beast::Severity::Disabled}; + SilentPerfLog perfLog; + std::shared_ptr collector{std::make_shared()}; + JobQueue queue; + + explicit GaugeFixture(int threadCount) + : queue(threadCount, collector, logs.journal("JobQueue"), logs, perfLog) + { + } + + /** + * Publish one collection pass, then read a gauge by job type. + */ + [[nodiscard]] std::optional + read(JobType type, char const* suffix) const + { + collector->runHooks(); + return collector->gaugeValue(JobTypes::name(type) + suffix); + } + }; + + /** + * A gauge exists for a limited job type and not for a special one. + * + * Creation is observed through the RecordingCollector: a name no gauge + * was created for is absent from its value map even after a collection + * pass, whereas a created one is present. That distinguishes "never + * created" from "created and reading 0", which a value check alone + * cannot. + */ + void + testGaugeCreation() + { + testcase("Saturation gauge creation"); + + GaugeFixture fixture(1); + + // JtLedgerReq has limit 3, so it is not special and must be gauged. + BEAST_EXPECT(!JobTypes::instance().get(JtLedgerReq).special()); + BEAST_EXPECT(JobTypes::instance().get(JtLedgerReq).limit() == 3); + BEAST_EXPECT(fixture.read(JtLedgerReq, kSuffixWaiting).has_value()); + BEAST_EXPECT(fixture.read(JtLedgerReq, kSuffixRunning).has_value()); + BEAST_EXPECT(fixture.read(JtLedgerReq, kSuffixDeferred).has_value()); + + // A quiescent queue publishes exactly zero, not "no value". + BEAST_EXPECT(fixture.read(JtLedgerReq, kSuffixWaiting) == 0u); + BEAST_EXPECT(fixture.read(JtLedgerReq, kSuffixRunning) == 0u); + BEAST_EXPECT(fixture.read(JtLedgerReq, kSuffixDeferred) == 0u); + + // JtPeer is special (limit_ == 0) and must have no gauges at all. + BEAST_EXPECT(JobTypes::instance().get(JtPeer).special()); + BEAST_EXPECT(JobTypes::instance().get(JtPeer).limit() == 0); + BEAST_EXPECT(!fixture.read(JtPeer, kSuffixWaiting).has_value()); + BEAST_EXPECT(!fixture.read(JtPeer, kSuffixRunning).has_value()); + BEAST_EXPECT(!fixture.read(JtPeer, kSuffixDeferred).has_value()); + } + + /** + * Driving a capped type past its limit reports exact running and + * deferred counts, and deferred returns to exactly 0 once drained. + * + * JtPack is used because its limit is 1, which makes both figures + * unambiguous: with the single slot occupied by a job that blocks until + * released, every further submission must defer. + */ + void + testDeferredGaugeExactValues() + { + testcase("Saturation gauge deferred counts"); + + int const limit = JobTypes::instance().get(JtPack).limit(); + BEAST_EXPECT(limit == 1); + + // More threads than the type's limit, so `running` is capped by the + // job type rather than by thread availability. + GaugeFixture fixture(4); + + // The first job blocks inside doJob() until released, holding the + // one slot. `entered` proves it really is running before the + // remaining jobs are submitted. + std::mutex mutex; + std::condition_variable cv; + bool release = false; + int entered = 0; + + auto blockingJob = [&]() { + std::unique_lock lock(mutex); + ++entered; + cv.notify_all(); + cv.wait(lock, [&release] { return release; }); + }; + + BEAST_EXPECT(fixture.queue.addJob(JtPack, "GaugeHold", blockingJob)); + { + std::unique_lock lock(mutex); + BEAST_EXPECT( + cv.wait_for(lock, std::chrono::seconds(10), [&entered] { return entered == 1; })); + } + + // Every one of these must defer: the limit is already reached. + int const extra = 4; + for (int i = 0; i < extra; ++i) + BEAST_EXPECT(fixture.queue.addJob(JtPack, "GaugeDefer", blockingJob)); + + // Exact values, not bounds: `running` equals the type's limit and + // `deferred` equals the number of submissions beyond it. Both are + // deterministic here because no JtPack job can complete until + // `release` is set. + BEAST_EXPECT(fixture.read(JtPack, kSuffixRunning) == static_cast(limit)); + BEAST_EXPECT(fixture.read(JtPack, kSuffixDeferred) == static_cast(extra)); + + // `waiting` counts everything submitted and not yet started, which + // is the deferred jobs -- the running one was decremented when + // getNextJob() picked it up. + BEAST_EXPECT(fixture.read(JtPack, kSuffixWaiting) == static_cast(extra)); + + // Cross-check against the public accessors, so a gauge that silently + // published a stale or unrelated number would be caught. + BEAST_EXPECT(fixture.queue.getJobCount(JtPack) == extra); + BEAST_EXPECT(fixture.queue.getJobCountTotal(JtPack) == extra + limit); + + // Release everything and drain. + { + std::scoped_lock const lock(mutex); + release = true; + } + cv.notify_all(); + fixture.queue.stop(); + BEAST_EXPECT(fixture.queue.isStopped()); + + // All five jobs ran, so the backlog is gone: deferred is exactly 0, + // and so are waiting and running. + BEAST_EXPECT(entered == extra + limit); + BEAST_EXPECT(fixture.read(JtPack, kSuffixDeferred) == 0u); + BEAST_EXPECT(fixture.read(JtPack, kSuffixWaiting) == 0u); + BEAST_EXPECT(fixture.read(JtPack, kSuffixRunning) == 0u); + } + + /** + * Negative path: an uncapped job type never reports non-zero deferred. + * + * JtClient's limit is std::numeric_limits::max(), so + * `addRefCountedJob()` can never take the `++data.deferred` branch. The + * gauge must therefore read exactly 0 both while jobs are in flight and + * after the queue drains -- otherwise a dashboard would attribute + * backpressure to a type that cannot experience it. + */ + void + testUncappedTypeNeverDefers() + { + testcase("Saturation gauge uncapped type"); + + int const limit = JobTypes::instance().get(JtClient).limit(); + BEAST_EXPECT(limit == std::numeric_limits::max()); + BEAST_EXPECT(!JobTypes::instance().get(JtClient).special()); + + // One thread, so submissions greatly outnumber the workers that can + // service them. Under a capped type this would defer; here it must + // not, which is what separates "waiting" from "deferred". + GaugeFixture fixture(1); + + std::mutex mutex; + std::condition_variable cv; + bool release = false; + int entered = 0; + + int const jobs = 6; + for (int i = 0; i < jobs; ++i) + { + BEAST_EXPECT(fixture.queue.addJob(JtClient, "GaugeUncapped", [&]() { + std::unique_lock lock(mutex); + ++entered; + cv.notify_all(); + cv.wait(lock, [&release] { return release; }); + })); + } + + // At least one job is in flight and the rest are backlogged, yet + // deferred stays at exactly 0 because the type has no limit. + { + std::unique_lock lock(mutex); + BEAST_EXPECT( + cv.wait_for(lock, std::chrono::seconds(10), [&entered] { return entered >= 1; })); + } + BEAST_EXPECT(fixture.read(JtClient, kSuffixDeferred) == 0u); + + { + std::scoped_lock const lock(mutex); + release = true; + } + cv.notify_all(); + fixture.queue.stop(); + + BEAST_EXPECT(entered == jobs); + BEAST_EXPECT(fixture.read(JtClient, kSuffixDeferred) == 0u); + BEAST_EXPECT(fixture.read(JtClient, kSuffixWaiting) == 0u); + BEAST_EXPECT(fixture.read(JtClient, kSuffixRunning) == 0u); + } + + /** + * Exactly the non-special job types are gauged, three gauges each, and + * every published value is non-negative. + * + * The coverage half pins the cardinality the metric family adds: one + * gauge per non-special type per counter and none for special types, so + * a job type gaining or losing a limit shows up here. + * + * The non-negativity half is the observable consequence of the clamp in + * `collect()`. The clamp itself cannot be triggered from a test: it + * guards `waiting` / `running` / `deferred`, which are private to + * JobQueue and only ever incremented and decremented in matched pairs, + * so forcing one negative would need the internals hacked. What is + * assertable is the property the clamp exists to guarantee -- since + * `Gauge::value_type` is unsigned, an unclamped negative would surface + * as a value near 2^64 rather than as a small number, which is exactly + * what the upper bound below rules out. + */ + void + testGaugeCoverageAndNonNegative() + { + testcase("Saturation gauge coverage"); + + GaugeFixture fixture(1); + fixture.collector->runHooks(); + + // Sanity-check the fixture against the job-type table itself, so the + // expected counts are derived rather than hard-coded. + int nonSpecial = 0; + int special = 0; + int gauges = 0; + bool allSmall = true; + + // No job has been submitted, so every published value must be 0. + // The bound is deliberately generous: it is here to catch an + // unsigned wrap, not to re-assert the exact zero above. + auto const kWrapGuard = static_cast(1) << 32; + + for (auto const& [type, info] : JobTypes::instance()) + { + if (type == JtInvalid) + continue; + + info.special() ? ++special : ++nonSpecial; + + for (char const* suffix : {kSuffixWaiting, kSuffixRunning, kSuffixDeferred}) + { + auto const value = fixture.collector->gaugeValue(info.name() + suffix); + + // Presence must agree with speciality, in both directions. + BEAST_EXPECT(value.has_value() == !info.special()); + if (!value) + continue; + + ++gauges; + if (*value >= kWrapGuard) + allSmall = false; + BEAST_EXPECT(*value == 0u); + } + } + + BEAST_EXPECT(nonSpecial == 35); + BEAST_EXPECT(special == 11); + BEAST_EXPECT(gauges == nonSpecial * 3); + BEAST_EXPECT(gauges == 105); + BEAST_EXPECT(allSmall); + } + public: void run() override { testAddJob(); testPostCoro(); + testGaugeCreation(); + testGaugeCoverageAndNonNegative(); + testDeferredGaugeExactValues(); + testUncappedTypeNeverDefers(); } }; diff --git a/src/test/core/Workers_test.cpp b/src/test/core/Workers_test.cpp index fe3820b84a1..102f5df14d6 100644 --- a/src/test/core/Workers_test.cpp +++ b/src/test/core/Workers_test.cpp @@ -36,13 +36,14 @@ class PerfLogTest : public PerfLog } void - jobQueue(JobType const type) override + jobQueue(JobType const type, std::string const& name) override { } void jobStart( JobType const type, + std::string const& name, std::chrono::microseconds dur, std::chrono::time_point startTime, int instance) override @@ -50,7 +51,11 @@ class PerfLogTest : public PerfLog } void - jobFinish(JobType const type, std::chrono::microseconds dur, int instance) override + jobFinish( + JobType const type, + std::string const& name, + std::chrono::microseconds dur, + int instance) override { } diff --git a/src/test/csf/Peer.h b/src/test/csf/Peer.h index 79bffec9cb3..dba528886c9 100644 --- a/src/test/csf/Peer.h +++ b/src/test/csf/Peer.h @@ -16,6 +16,10 @@ #include #include +#ifdef XRPL_ENABLE_TELEMETRY +#include +#endif + #include #include #include @@ -24,6 +28,7 @@ #include #include #include +#include #include #include @@ -35,6 +40,7 @@ #include #include #include +#include #include #include @@ -685,6 +691,47 @@ struct Peer { } + // Telemetry hooks — no-ops in the simulator. The generic engine calls + // these at every phase transition / outcome resolution so the + // production adaptor (RCLConsensus::Adaptor) can record events on the + // round span; the simulator runs without telemetry. + void + onPhaseEvent(std::string_view, std::string_view) + { + } + + void + onOutcomeEvent(std::string_view) + { + } + + // The generic engine parents its phase spans under the round span via + // this context; the simulator runs without telemetry, so return an + // invalid context and no phase span is created. + static telemetry::SpanContext + roundSpanContext() + { + return {}; + } + +#ifdef XRPL_ENABLE_TELEMETRY + /** + * Provide telemetry access for the Consensus template. + * + * The test Peer adaptor uses a static disabled NullTelemetry instance + * so that all shouldTrace*() checks return false and no spans are + * created during simulation tests. It is static because the shared + * disabled instance does not depend on any per-peer state. + */ + static telemetry::Telemetry& + getTelemetry() + { + static auto tel = telemetry::makeTelemetry( + telemetry::Telemetry::Setup{}, beast::Journal{beast::Journal::getNullSink()}); + return *tel; + } +#endif + // Share a message by broadcasting to all connected peers template void diff --git a/src/test/overlay/TMGetObjectByHash_test.cpp b/src/test/overlay/TMGetObjectByHash_test.cpp index e579989181f..b0e3b70dfad 100644 --- a/src/test/overlay/TMGetObjectByHash_test.cpp +++ b/src/test/overlay/TMGetObjectByHash_test.cpp @@ -20,7 +20,9 @@ #include #include #include +#include #include +#include #include #include @@ -33,6 +35,9 @@ #include #include +#include +#include +#include #include #include @@ -100,6 +105,50 @@ class TMGetObjectByHash_test : public beast::unit_test::Suite return lastSentMessage_; } + /** + * Capture the charge the handler applies, then apply it for real. + * + * `Peer::charge()` is pure virtual and `processGetObjectByHash()` + * calls it unqualified, so this override sees the exact + * `Resource::Charge` the handler built -- the same object passed to + * `computeGetObjectByHashFee()`'s caller. That makes the handler's + * *choice of argument* observable, which reading `fee_` or calling + * the pricing helper with the test's own arguments cannot do. + * + * Recorded before forwarding so the base-class strand hop cannot + * reorder the observation; forwarding keeps the production + * disconnect/accounting behaviour intact. + */ + void + charge(Resource::Charge const& fee, std::string const& context) override + { + lastAppliedCharge_ = fee; + lastChargeContext_ = context; + PeerImp::charge(fee, context); + } + + /** + * The charge captured by the override above, or nullopt if none. + * + * `Resource::Charge` has no default constructor, so the optional + * also distinguishes "not charged at all" from "charged zero" -- + * a distinction the rejection-gate tests depend on. + */ + [[nodiscard]] std::optional const& + getLastAppliedCharge() const + { + return lastAppliedCharge_; + } + + /** + * The context string that accompanied the captured charge. + */ + [[nodiscard]] std::string const& + getLastChargeContext() const + { + return lastChargeContext_; + } + // Synchronous test access to the JobQueue-dispatched processor. // The production path runs this on JtLedgerReq; tests need a // synchronous entry point to inspect the reply via send(). @@ -111,6 +160,25 @@ class TMGetObjectByHash_test : public beast::unit_test::Suite processGetObjectByHash(m); } + // Read the accumulated per-message charge. `currentFeeCharge()` is + // protected on PeerImp; exposed here because it is the one + // deterministic, same-thread witness that a rejection gate fired -- + // `charge()` itself dispatches to the peer's strand. + [[nodiscard]] Resource::Charge + peekFeeCharge() const + { + return currentFeeCharge(); + } + + // The differential-pricing helper, so a test can compare the charge + // applied by the handler against the helper's own result for the + // same inputs. Static and protected on PeerImp. + [[nodiscard]] static Resource::Charge + peekComputeFee(int const requested, int const found) + { + return computeGetObjectByHashFee(requested, found); + } + static void resetId() { @@ -119,12 +187,55 @@ class TMGetObjectByHash_test : public beast::unit_test::Suite private: inline static Peer::id_t id = 0; + + /** + * The last message handed to send(). + * + * @note Not synchronised. When the handler runs on a JobQueue + * worker (the `onMessage()` tests), this is written on that worker + * and read on the test thread, so every such test must call + * `env.app().getJobQueue().rendezvous()` before reading it. The + * rendezvous supplies the happens-before edge: the worker's + * `--processCount_` under `mutex_` in `JobQueue::processTask()` + * releases, and the waiter's predicate acquires the same mutex. + * Tests that drive `runProcessGetObjectByHash()` directly run + * wholly on the test thread and need no rendezvous. + */ std::shared_ptr lastSentMessage_; + + /** + * @see getLastAppliedCharge(). Same threading rules as above. + */ + std::optional lastAppliedCharge_; + + /** + * @see getLastChargeContext(). Same threading rules as above. + */ + std::string lastChargeContext_; }; shared_context context_{makeSslContext("")}; ProtocolVersion protocolVersion_{1, 7}; + /** + * Seed offset for hashes that must NOT be present in the NodeStore. + * + * `createRequest()` stores `sha512Half(i)` for i in [0, numObjects), and + * numObjects can reach kHardMaxReplyNodes. Offsetting well past that + * keeps "unstored" hashes genuinely absent. + */ + static constexpr int kUnstoredHashSeed = 1'000'000; + + /** + * Build a live PeerTest registered with the overlay. + * + * @note `overlay.addActive()` stores only `std::weak_ptr`s + * (`OverlayImpl::peers_`, `ids_` and `list_` are all weak), so it does + * *not* keep the peer alive. The returned `shared_ptr` is the sole + * owner; keep it in scope for the whole test. Safety for the + * JobQueue-dispatched path comes from the job lambda locking its own + * `weak_ptr` plus the `rendezvous()` each such test performs. + */ std::shared_ptr createPeer(jtx::Env& env) { @@ -187,6 +298,27 @@ class TMGetObjectByHash_test : public beast::unit_test::Suite return request; } + /** + * Parse the reply captured by PeerTest::send(). + * + * @return The decoded message, or std::nullopt when nothing was sent. + */ + std::optional + parseReply(std::shared_ptr const& peer) + { + auto const sentMessage = peer->getLastSentMessage(); + if (!sentMessage) + return std::nullopt; + + auto const& buffer = sentMessage->getBuffer(compression::Compressed::Off); + BEAST_EXPECT(buffer.size() > 6); + + // Skip the 6-byte message header (4 size + 2 type). + protocol::TMGetObjectByHash reply; + BEAST_EXPECT(reply.ParseFromArray(buffer.data() + 6, buffer.size() - 6) == true); + return reply; + } + /** * Test that reply is limited to hardMaxReplyNodes when more objects * are requested than the limit allows. @@ -209,28 +341,675 @@ class TMGetObjectByHash_test : public beast::unit_test::Suite peer->runProcessGetObjectByHash(request); // Verify that a reply was sent - auto sentMessage = peer->getLastSentMessage(); - BEAST_EXPECT(sentMessage != nullptr); + auto reply = parseReply(peer); + BEAST_EXPECT(reply.has_value()); + if (!reply) + return; - // Parse the reply message - auto const& buffer = sentMessage->getBuffer(compression::Compressed::Off); + // Verify the reply is limited to expectedReplySize + BEAST_EXPECT(reply->objects_size() == expectedReplySize); + } - BEAST_EXPECT(buffer.size() > 6); - // Skip the message header (6 bytes: 4 for size, 2 for type) - protocol::TMGetObjectByHash reply; - BEAST_EXPECT(reply.ParseFromArray(buffer.data() + 6, buffer.size() - 6) == true); + //-------------------------------------------------------------------------- + // Request-gate rejection paths + //-------------------------------------------------------------------------- - // Verify the reply is limited to expectedReplySize - BEAST_EXPECT(reply.objects_size() == expectedReplySize); + /** + * Build a query request with @p numObjects hashes and nothing stored. + * + * Distinct from `createRequest()`, which writes every hash to the + * NodeStore. The rejection gates return before any NodeStore access, so + * storing 12289 objects to test them would cost real time and prove + * nothing. Hashes are derived from the index but need not resolve. + * + * @param numObjects Objects to place in the request. + * @param type Message type; must not be otFETCH_PACK or + * otTRANSACTIONS, both of which are intercepted by + * earlier branches of onMessage(). + */ + static std::shared_ptr + createUnstoredRequest( + int const numObjects, + protocol::TMGetObjectByHash::ObjectType const type = + protocol::TMGetObjectByHash_ObjectType_otLEDGER) + { + auto request = std::make_shared(); + request->set_type(type); + request->set_query(true); + + for (int i = 0; i < numObjects; ++i) + { + // Offset the seed so these hashes cannot collide with the ones + // createRequest() stores, keeping "unstored" unambiguous. + uint256 const hash(xrpl::sha512Half(i + kUnstoredHashSeed)); + auto* object = request->add_objects(); + object->set_hash(hash.data(), hash.size()); + } + return request; + } + + /** + * An oversized request is refused with no reply and an exact fee. + * + * This is the gate that `getobject_rejected_total{reason="oversize"}` + * counts. The counter itself is not readable in-process (see the note + * on run()), so the assertions are on the two observable effects of the + * same early return: no message was sent, and `fee_` holds exactly + * `kFeeInvalidData`. + */ + void + testOversizeRejection() + { + testcase("Oversize Rejection"); + + Env env(*this); + PeerTest::resetId(); + auto peer = createPeer(env); + + // Successful-setup assertion: a fresh peer starts at the trivial + // fee, so the post-condition below can only come from this call. + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeTrivialPeer.cost()); + BEAST_EXPECT(peer->getLastSentMessage() == nullptr); + + int const oversize = static_cast(Tuning::kHardMaxReplyNodes) + 1; + peer->onMessage(createUnstoredRequest(oversize)); + + // State: nothing was replied to, because the gate returns before + // the job is queued. + BEAST_EXPECT(peer->getLastSentMessage() == nullptr); + + // Cause: the charge is exactly the invalid-data fee, not merely + // "some larger fee". The label pins which gate fired -- the + // malformed-ledgerhash gate charges a different constant. + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeInvalidData.cost()); + BEAST_EXPECT(peer->peekFeeCharge().cost() == 400); + BEAST_EXPECT(peer->peekFeeCharge().label() == Resource::kFeeInvalidData.label()); + + // Negative path for the differential charge: the gate returns before + // the handler runs, so computeGetObjectByHashFee() is never reached + // and charge() is never called. Nothing was applied at all -- which + // the optional distinguishes from a zero-cost charge. + BEAST_EXPECT(!peer->getLastAppliedCharge().has_value()); + } + + /** + * Exactly at the limit the request is accepted, so the gate is a strict + * `>` and not `>=`. + * + * Negative control for testOversizeRejection: without it, a gate that + * rejected everything would pass that test. + */ + void + testAtLimitNotRejected() + { + testcase("At Limit Not Rejected"); + + Env env(*this); + PeerTest::resetId(); + auto peer = createPeer(env); + + int const atLimit = static_cast(Tuning::kHardMaxReplyNodes); + peer->onMessage(createUnstoredRequest(atLimit)); + + // Accepted: the request was queued, so the fee is the + // moderate-burden admission charge, not the invalid-data charge. + // + // `fee_.update()` for the admission charge runs on this thread, but + // the enqueued worker also writes the peer's reply. Drain the queue + // before reading anything so the observation cannot race the worker. + env.app().getJobQueue().rendezvous(); + + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeModerateBurdenPeer.cost()); + BEAST_EXPECT(peer->peekFeeCharge().cost() != Resource::kFeeInvalidData.cost()); + BEAST_EXPECT(peer->peekFeeCharge().cost() == 250); + + // The request was in bounds, so the worker ran and replied. Nothing + // was stored, so every lookup missed and the reply is empty. + auto reply = parseReply(peer); + BEAST_EXPECT(reply.has_value()); + if (reply) + BEAST_EXPECT(reply->objects_size() == 0); + + // Positive counterpart to the oversize test: because the handler did + // run, a differential charge was applied, and it is exactly the + // all-miss price for the full request size. + auto const& applied = peer->getLastAppliedCharge(); + BEAST_EXPECT(applied.has_value()); + if (applied) + { + BEAST_EXPECT(applied->cost() == PeerTest::peekComputeFee(atLimit, 0).cost()); + BEAST_EXPECT(applied->cost() == 99176); + } + } + + /** + * A wrong-sized ledgerhash is refused with no reply and an exact fee. + * + * This is the gate `getobject_rejected_total{reason="malformed_ledgerhash"}` + * counts. `stringIsUInt256Sized` requires exactly `uint256::size()` + * bytes, so both a short and a long hash must be refused; a test using + * only one would miss an off-by-one in either direction. + * + * @param hashSize Byte length of the malformed ledgerhash. + */ + void + testMalformedLedgerHashRejection(std::size_t const hashSize) + { + testcase("Malformed LedgerHash Rejection: " + std::to_string(hashSize) + " bytes"); + + BEAST_EXPECT(hashSize != uint256::size()); + + Env env(*this); + PeerTest::resetId(); + auto peer = createPeer(env); + + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeTrivialPeer.cost()); + + // Small in-bounds object count, so this can only be the ledgerhash + // gate: the oversize gate is checked afterwards and cannot fire. + auto request = createUnstoredRequest(1); + request->set_ledgerhash(std::string(hashSize, 'x')); + peer->onMessage(request); + + BEAST_EXPECT(peer->getLastSentMessage() == nullptr); + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeMalformedRequest.cost()); + BEAST_EXPECT(peer->peekFeeCharge().cost() == 200); + BEAST_EXPECT(peer->peekFeeCharge().label() == Resource::kFeeMalformedRequest.label()); + + // Negative path: the gate returns before the handler, so no + // differential charge was ever applied. + BEAST_EXPECT(!peer->getLastAppliedCharge().has_value()); + } + + /** + * A correctly sized ledgerhash passes the gate. + * + * Negative control for testMalformedLedgerHashRejection. + */ + void + testWellFormedLedgerHashAccepted() + { + testcase("Well-Formed LedgerHash Accepted"); + + Env env(*this); + PeerTest::resetId(); + auto peer = createPeer(env); + + auto request = createUnstoredRequest(1); + uint256 const ledgerHash(xrpl::sha512Half(0)); + BEAST_EXPECT(ledgerHash.size() == uint256::size()); + request->set_ledgerhash(ledgerHash.data(), ledgerHash.size()); + peer->onMessage(request); + + // Drain the enqueued worker before observing, as in + // testAtLimitNotRejected. + env.app().getJobQueue().rendezvous(); + + // Not the malformed charge: the request was admitted. + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeModerateBurdenPeer.cost()); + BEAST_EXPECT(peer->peekFeeCharge().cost() != Resource::kFeeMalformedRequest.cost()); + BEAST_EXPECT(peer->peekFeeCharge().cost() == 250); + + // A reply was produced, and it echoes the request's ledgerhash. + auto reply = parseReply(peer); + BEAST_EXPECT(reply.has_value()); + if (reply) + { + BEAST_EXPECT(reply->has_ledgerhash()); + BEAST_EXPECT(reply->ledgerhash() == request->ledgerhash()); + } + } + + //-------------------------------------------------------------------------- + // Hit / miss split and charge + //-------------------------------------------------------------------------- + + /** + * Build a request that interleaves stored and unstored hashes. + * + * One of each is taken in turn until a side runs out, then whichever + * side remains is drained. Interleaving matters: a handler that stopped + * at the first miss would return fewer objects than expected, which a + * stored-then-unstored layout could hide. + * + * Asserts on its own setup, so a miscount here is reported at this call + * rather than as a confusing reply-size failure later. + * + * @param env Environment whose NodeStore receives the writes. + * @param numStored Hashes written to the NodeStore, i.e. hits. + * @param numUnstored Hashes left absent, i.e. misses. + * @param storedHashes Out-param populated with the stored hashes. + * @return The assembled request. + */ + std::shared_ptr + buildInterleavedRequest( + Env& env, + int const numStored, + int const numUnstored, + std::set& storedHashes) + { + auto& nodeStore = env.app().getNodeStore(); + + auto request = std::make_shared(); + request->set_type(protocol::TMGetObjectByHash_ObjectType_otLEDGER); + request->set_query(true); + + int stored = 0; + int unstored = 0; + for (int i = 0; i < numStored + numUnstored; ++i) + { + // Alternate while both remain; then drain whichever is left. + bool const takeStored = + (stored < numStored) && (unstored >= numUnstored || (i % 2) == 0); + + uint256 const hash( + xrpl::sha512Half(takeStored ? stored : unstored + kUnstoredHashSeed)); + + if (takeStored) + { + Blob data(100, static_cast(stored % 256)); + nodeStore.store( + NodeObjectType::Ledger, std::move(data), hash, nodeStore.earliestLedgerSeq()); + BEAST_EXPECT(storedHashes.insert(hash).second); + ++stored; + } + else + { + ++unstored; + } + + auto* object = request->add_objects(); + object->set_hash(hash.data(), hash.size()); + } + + // Setup assertions: the mix is exactly what was asked for. + BEAST_EXPECT(stored == numStored); + BEAST_EXPECT(unstored == numUnstored); + BEAST_EXPECT(storedHashes.size() == static_cast(numStored)); + BEAST_EXPECT(request->objects_size() == numStored + numUnstored); + + return request; + } + + /** + * Every replied object is a distinct hash drawn from @p storedHashes. + * + * Without the distinctness check a handler that returned the same hit + * twice would still satisfy a reply-size assertion. + * + * @param reply The decoded reply. + * @param storedHashes The hashes that were written to the NodeStore. + * @param numStored Expected number of distinct returned hashes. + */ + void + verifyReplyObjects( + protocol::TMGetObjectByHash const& reply, + std::set const& storedHashes, + int const numStored) + { + std::set returned; + for (int i = 0; i < reply.objects_size(); ++i) + { + auto const& obj = reply.objects(i); + BEAST_EXPECT(obj.hash().size() == uint256::size()); + BEAST_EXPECT(returned.insert(uint256::fromRaw(obj.hash())).second); + BEAST_EXPECT(storedHashes.contains(uint256::fromRaw(obj.hash()))); + } + BEAST_EXPECT(returned.size() == static_cast(numStored)); + } + + /** + * A mixed request returns exactly the stored objects and nothing else. + * + * This is the split `getobject_lookups_total{result=hit|miss}` records: + * the handler derives the miss count as `requested - found`, so an + * exact reply size is exactly the hit count the metric would report. + * + * @param numStored Hashes written to the NodeStore before the call. + * @param numUnstored Hashes that will miss. + */ + void + testHitMissSplit(int const numStored, int const numUnstored) + { + testcase( + "Hit/Miss Split: " + std::to_string(numStored) + " stored, " + + std::to_string(numUnstored) + " unstored"); + + Env env(*this); + PeerTest::resetId(); + auto peer = createPeer(env); + + std::set storedHashes; + auto request = buildInterleavedRequest(env, numStored, numUnstored, storedHashes); + + int const requested = numStored + numUnstored; + peer->runProcessGetObjectByHash(request); + + auto reply = parseReply(peer); + BEAST_EXPECT(reply.has_value()); + if (!reply) + return; + + // The exact hit count. Every stored hash is returned and no + // unstored one is, so hits == numStored and the derived miss count + // is exactly numUnstored. + BEAST_EXPECT(reply->objects_size() == numStored); + BEAST_EXPECT(requested - reply->objects_size() == numUnstored); + + verifyReplyObjects(*reply, storedHashes, numStored); + + // Cause: the value recorded as getobject_charge is exactly the + // charge the handler applied, captured by PeerTest::charge(). + auto const& applied = peer->getLastAppliedCharge(); + BEAST_EXPECT(applied.has_value()); + if (!applied) + return; + BEAST_EXPECT(applied->cost() == PeerTest::peekComputeFee(requested, numStored).cost()); + BEAST_EXPECT(applied->label() == "GetObject differential"); + + // These request sizes are all within kFreeObjectsPerRequest, so the + // charge is exactly zero regardless of the split. Asserted rather + // than assumed: it is why this test does not also pin a non-trivial + // fee -- testComputeFeeExactValues covers the billable bands. + BEAST_EXPECT(requested <= static_cast(Tuning::kFreeObjectsPerRequest)); + BEAST_EXPECT(applied->cost() == 0); + + // `fee_` is untouched on this path: the handler charges through + // charge(), never through fee_.update(). + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeTrivialPeer.cost()); + } + + /** + * One pricing case: inputs, the derived expectation, and the literal. + * + * Both expectations are kept. `derived` is written from the Tuning + * constants so a deliberate re-pricing needs one edit; `literal` is the + * number as of this branch so a re-pricing cannot pass unnoticed by + * being self-consistently wrong. + */ + struct FeeCase + { + /** + * Objects the peer asked for. + */ + int requested; + /** + * Objects that resolved in the NodeStore. + */ + int found; + /** + * Expectation computed from the Tuning constants. + */ + int derived; + /** + * The same value as a hard number. + */ + int literal; + /** + * Reported when the case fails. + */ + char const* why; + }; + + /** + * Assert computeGetObjectByHashFee() equals both expectations of a case. + * + * @param fc The case to check. + */ + void + checkFeeCase(FeeCase const& fc) + { + auto const cost = PeerTest::peekComputeFee(fc.requested, fc.found).cost(); + BEAST_EXPECTS(cost == fc.derived, fc.why); + BEAST_EXPECTS(cost == fc.literal, fc.why); + } + + /** + * The Tuning constants the fee expectations are built from. + * + * Verified against their literal values by testComputeFeeExactValues() + * so a silent re-pricing shows up as a failure there rather than as a + * self-consistent but wrong expectation in every case below. + */ + struct FeeConstants + { + int free{static_cast(Tuning::kFreeObjectsPerRequest)}; + int hit{static_cast(Tuning::kCostPerLookupHit)}; + int miss{static_cast(Tuning::kCostPerLookupMiss)}; + int bandSmall{static_cast(Tuning::kCostBandSmall)}; + int bandMedium{static_cast(Tuning::kCostBandMedium)}; + int bandLarge{static_cast(Tuning::kCostBandLarge)}; + int smallMax{static_cast(Tuning::kBandSmallMax)}; + int mediumMax{static_cast(Tuning::kBandMediumMax)}; + }; + + /** + * Every pricing case, in one table. + * + * @param k The Tuning constants to derive expectations from. + */ + static std::vector + makeFeeCases(FeeConstants const& k) + { + return { + // Wholly free: at or below the free allowance nothing is + // billable, so only the small size band applies -- which is 0. + {k.free, k.free, k.bandSmall, 0, "at the free allowance"}, + {0, 0, k.bandSmall, 0, "empty request"}, + {1, 1, k.bandSmall, 0, "one object, one hit"}, + + // All hits, one object past the allowance: one billable hit. + {k.free + 1, k.free + 1, k.hit + k.bandSmall, 1, "one billable hit"}, + + // All misses, one past the allowance: misses are billed first, + // so the single billable object is priced as a miss, not a hit. + {k.free + 1, 0, k.miss + k.bandSmall, 8, "one billable miss"}, + + // Mixed at the small-band edge: 64 requested, 32 found. + // Billable is 64-16 = 48; misses are 32 and all billable, + // leaving 16 billable hits. + {k.smallMax, + 32, + (16 * k.hit) + (32 * k.miss) + k.bandSmall, + 272, + "small-band edge, mixed"}, + + // One past the small band moves to the medium surcharge. + {k.smallMax + 1, + k.smallMax + 1, + ((k.smallMax + 1 - k.free) * k.hit) + k.bandMedium, + 149, + "first medium-band size"}, + + // The medium band's last size, then one past it, which moves to + // the large surcharge. + {k.mediumMax, + k.mediumMax, + ((k.mediumMax - k.free) * k.hit) + k.bandMedium, + 1108, + "last medium-band size"}, + {k.mediumMax + 1, + k.mediumMax + 1, + ((k.mediumMax + 1 - k.free) * k.hit) + k.bandLarge, + 2009, + "first large-band size"}, + + // Clamp: found > requested cannot make the miss count negative, + // so the fee is the same as the all-hit case (1). + {k.free + 1, k.free + 10, k.hit + k.bandSmall, 1, "found exceeds requested"}, + }; + } + + /** + * computeGetObjectByHashFee() returns exactly the documented value. + * + * The metric records the helper's result verbatim, so pinning the helper + * pins what `getobject_charge` reports. + */ + void + testComputeFeeExactValues() + { + testcase("Compute Fee Exact Values"); + + FeeConstants const k; + + // Verify the constants themselves, so a silent re-pricing shows up + // here rather than as a self-consistent but wrong expectation. + BEAST_EXPECT(k.free == 16); + BEAST_EXPECT(k.hit == 1); + BEAST_EXPECT(k.miss == 8); + BEAST_EXPECT(k.bandSmall == 0); + BEAST_EXPECT(k.bandMedium == 100); + BEAST_EXPECT(k.bandLarge == 1000); + BEAST_EXPECT(k.smallMax == 64); + BEAST_EXPECT(k.mediumMax == 1024); + + auto const cases = makeFeeCases(k); + BEAST_EXPECT(cases.size() == 10); + for (auto const& fc : cases) + checkFeeCase(fc); + + // A miss costs strictly more than a hit for the same request size. + // Relational, so it cannot be expressed as a table row. + BEAST_EXPECT( + PeerTest::peekComputeFee(k.free + 1, 0).cost() > + PeerTest::peekComputeFee(k.free + 1, k.free + 1).cost()); + + // The label is fixed, so a charge can be attributed to this helper. + BEAST_EXPECT(PeerTest::peekComputeFee(k.free + 1, 0).label() == "GetObject differential"); + } + + /** + * The charge the handler *applies* is priced on `requested`, not on the + * capped iteration count and not on `found`. + * + * Load-bearing because `getobject_charge` records the applied value: if + * `processGetObjectByHash()` priced on `iterLimit` -- i.e. + * `min(requested, kHardMaxReplyNodes)` -- the metric would under-report + * abusive batches by exactly the overshoot. + * + * The assertion is on `PeerTest::charge()`, which overrides the virtual + * the handler calls, so it observes the very `Resource::Charge` object + * the handler constructed. Two other candidate witnesses were rejected: + * - `fee_` / `peekFeeCharge()`: this path never touches `fee_`, it + * goes through `charge()`. + * - `usage_.balance()`: `Entry::add()` returns + * `localBalance.add(...) + remoteBalance` and `DecayingSample::add()` + * returns `value_ / Window` with `Window == kDecayWindowSeconds == + * 32`, so the balance is the applied cost divided by 32 with integer + * truncation. 99184 / 32 and 99176 / 32 are both 3099, so the + * balance cannot distinguish the mutation this test exists to catch. + * It also decays with wall-clock time (`BasicSecondsClock`), making + * any exact expectation racy. + */ + void + testChargeUsesRequestedCount() + { + testcase("Charge Uses Requested Count"); + + Env env(*this); + PeerTest::resetId(); + auto peer = createPeer(env); + + int const requested = static_cast(Tuning::kHardMaxReplyNodes) + 1; + int const capped = static_cast(Tuning::kHardMaxReplyNodes); + + // Successful-setup assertion: nothing has been charged yet, so the + // post-condition below can only come from the handler call. + BEAST_EXPECT(!peer->getLastAppliedCharge().has_value()); + + // No hashes are stored, so every lookup misses and `found` is 0. + // Called directly, so the handler and the charge both run on this + // thread: `charge()` dispatches to strand_, and boost's strand + // `dispatch` runs the function inline when the caller is not already + // in the strand and the strand is idle. The capture in the override + // happens before that hop regardless, so the observation is + // deterministic either way. + peer->runProcessGetObjectByHash(createUnstoredRequest(requested)); + + auto reply = parseReply(peer); + BEAST_EXPECT(reply.has_value()); + if (!reply) + return; + BEAST_EXPECT(reply->objects_size() == 0); + + // State: a charge was applied at all. + auto const& applied = peer->getLastAppliedCharge(); + BEAST_EXPECT(applied.has_value()); + if (!applied) + return; + + // Cause: it is exactly the requested-count price. This is the + // assertion the test is named for. Under either plausible + // mis-pricing it reads a different number and therefore fails: + // priced on `iterLimit` (12288) -> 99176 + // priced on `found` (0) -> 0, since billable clamps to 0 + // and the band drops to Small + BEAST_EXPECT(applied->cost() == 99184); + BEAST_EXPECT(applied->cost() == PeerTest::peekComputeFee(requested, 0).cost()); + BEAST_EXPECT(applied->cost() != PeerTest::peekComputeFee(capped, 0).cost()); + + // Attribution: the charge came from the differential helper, not + // from one of the flat admission or rejection constants. + BEAST_EXPECT(applied->label() == "GetObject differential"); + BEAST_EXPECT(peer->getLastChargeContext() == "processed get object by hash request"); + + // `fee_` is untouched on this path, which is why the override above + // exists rather than a peekFeeCharge() assertion. + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeTrivialPeer.cost()); + + // Pricing on the requested count is strictly more expensive than + // pricing on the capped count, which is what makes the choice + // observable at all. + BEAST_EXPECT( + PeerTest::peekComputeFee(requested, 0).cost() > + PeerTest::peekComputeFee(capped, 0).cost()); + + // Exact values for both, so a change to either input is caught. + BEAST_EXPECT(PeerTest::peekComputeFee(requested, 0).cost() == 99184); + BEAST_EXPECT(PeerTest::peekComputeFee(capped, 0).cost() == 99176); } void run() override { + // NOTE ON METRIC COVERAGE. The five getobject_* instruments are + // recorded through the XRPL_METRIC_* macros, which push into the + // OpenTelemetry SDK. That API is write-only by design -- there is no + // read-back accessor and no in-memory metric reader in this build -- + // and a default jtx::Env leaves telemetry disabled, so the macros do + // not execute at all here. These tests therefore assert the + // observable behaviour of each instrumented code path, which pins + // the values the instruments are fed: + // getobject_request_objects <- the request's objects_size() + // getobject_lookups_total <- reply size (hits) and the derived + // miss count, per testHitMissSplit + // getobject_charge <- the applied Resource::Charge, + // captured by PeerTest::charge() + // getobject_rejected_total <- the two gates' exact fee_ values + // plus "no charge was applied" + // Only getobject_lookup_us has no in-process witness, being a wall + // clock reading. The counter and histogram values themselves remain + // unverified by unit test and are checked live against Prometheus + // per the design's live-validation step. int const limit = static_cast(Tuning::kHardMaxReplyNodes); testReplyLimit(limit + 1, limit); testReplyLimit(limit, limit); testReplyLimit(limit - 1, limit - 1); + + testOversizeRejection(); + testAtLimitNotRejected(); + testMalformedLedgerHashRejection(uint256::size() - 1); + testMalformedLedgerHashRejection(uint256::size() + 1); + testMalformedLedgerHashRejection(0); + testWellFormedLedgerHashAccepted(); + + testHitMissSplit(5, 3); + testHitMissSplit(0, 4); + testHitMissSplit(4, 0); + + testComputeFeeExactValues(); + testChargeUsesRequestedCount(); } }; diff --git a/src/tests/libxrpl/CMakeLists.txt b/src/tests/libxrpl/CMakeLists.txt index cafe72eff9b..743de4f043e 100644 --- a/src/tests/libxrpl/CMakeLists.txt +++ b/src/tests/libxrpl/CMakeLists.txt @@ -33,6 +33,7 @@ set(test_modules shamap tx protocol_autogen + telemetry ) if(NOT WIN32) list(APPEND test_modules net) @@ -63,4 +64,45 @@ if(verify_headers) verify_target_headers(xrpl_tests "${CMAKE_CURRENT_SOURCE_DIR}") endif() +# The telemetry test module exercises SpanGuard/Telemetry, which link against +# the OpenTelemetry SDK when the telemetry build option is enabled. +if(telemetry) + # The SpanGuardScope tests drive an InMemorySpanExporter to read exported + # spans (test-only; production never uses it). The Conan package ships + # libopentelemetry_exporter_in_memory.a but declares the component with no + # libs, so neither the umbrella target nor opentelemetry-cpp::exporter_in_memory + # puts the archive on the link line. Locate it by path and link it BEFORE + # the umbrella so its undefined SDK references resolve against the umbrella. + find_library( + OTEL_IN_MEMORY_EXPORTER_LIB + NAMES opentelemetry_exporter_in_memory + HINTS "${opentelemetry-cpp_PACKAGE_FOLDER_RELEASE}/lib" + REQUIRED + ) + target_link_libraries( + xrpl_tests + PRIVATE + "${OTEL_IN_MEMORY_EXPORTER_LIB}" + opentelemetry-cpp::opentelemetry-cpp + ) + # ValidationTracker lives in src/xrpld/ (not libxrpl), so we compile its + # implementation directly into the test binary and put src/ on the include + # path so its tests can reach headers. + target_include_directories(xrpl_tests PRIVATE ${CMAKE_SOURCE_DIR}/src) + target_sources( + xrpl_tests + PRIVATE + ${CMAKE_SOURCE_DIR}/src/xrpld/telemetry/detail/ValidationTracker.cpp + ) +else() + # MetricsRegistry lives in xrpld; compile its .cpp directly into the test + # target so the no-op path can be tested without linking all of xrpld. + # When telemetry=ON, XRPL_ENABLE_TELEMETRY is globally defined and the + # .cpp pulls in xrpld symbols we cannot satisfy here. + target_sources( + xrpl_tests + PRIVATE ${CMAKE_SOURCE_DIR}/src/xrpld/telemetry/MetricsRegistry.cpp + ) +endif() + gtest_discover_tests(xrpl_tests DISCOVERY_TIMEOUT 60) diff --git a/src/tests/libxrpl/helpers/TestServiceRegistry.h b/src/tests/libxrpl/helpers/TestServiceRegistry.h index 5475b54dc67..b11599310df 100644 --- a/src/tests/libxrpl/helpers/TestServiceRegistry.h +++ b/src/tests/libxrpl/helpers/TestServiceRegistry.h @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -337,6 +338,18 @@ class TestServiceRegistry : public ServiceRegistry throw std::logic_error("TestServiceRegistry::getPerfLog() not implemented"); } + telemetry::Telemetry& + getTelemetry() override + { + throw std::logic_error("TestServiceRegistry::getTelemetry() not implemented"); + } + + telemetry::MetricsRegistry* + getMetricsRegistry() override + { + return nullptr; + } + // Configuration and state bool isStopping() const override diff --git a/src/tests/libxrpl/telemetry/GetMeter.cpp b/src/tests/libxrpl/telemetry/GetMeter.cpp new file mode 100644 index 00000000000..902f40fcd99 --- /dev/null +++ b/src/tests/libxrpl/telemetry/GetMeter.cpp @@ -0,0 +1,112 @@ +/** + * @file GetMeter.cpp + * Unit tests for the direct OTel metrics API surface: + * - xrpl::telemetry::Telemetry::getMeter() on the disabled (noop) path. + * - The global MeterProvider contract that beast's OTelCollector shim + * relies on (GetMeterProvider()->GetMeter(kMeterName, kMeterVersion)). + * + * These tests deliberately exercise the global-provider level rather than a + * fully started TelemetryImpl: TelemetryImpl::start() spins up an OTLP HTTP + * exporter with background export threads and network connect attempts, which + * is unsuitable for a hermetic unit test. The design spec permits testing the + * metrics unlock at the provider level, which is the exact path the beast shim + * and getMeter() delegate to. + * + * Compiled only when XRPL_ENABLE_TELEMETRY is defined; the metrics API and SDK + * headers exist only in that build. + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace xrpl; +using namespace xrpl::telemetry; + +namespace metrics_api = opentelemetry::metrics; +namespace metrics_sdk = opentelemetry::sdk::metrics; + +// --------------------------------------------------------------- +// 1. Disabled path: Telemetry::getMeter() returns a usable noop meter. +// makeTelemetry() with enabled=false yields NullTelemetryOtel, whose +// getMeter() serves a meter from a process-wide NoopMeterProvider. The +// meter and any instrument built from it must be non-null, and Add() +// (positive and negative) must be inert but safe. +// --------------------------------------------------------------- +TEST(GetMeter, disabled_path_returns_usable_noop_meter) +{ + Telemetry::Setup setup; + setup.enabled = false; + + beast::Journal::Sink& sink = beast::Journal::getNullSink(); + beast::Journal const journal(sink); + auto telemetry = makeTelemetry(setup, journal); + ASSERT_NE(telemetry, nullptr); + + // getMeter() must return a concrete (non-null) meter, never a raw nullptr. + auto meter = telemetry->getMeter(); + ASSERT_TRUE(static_cast(meter)); + + // A synchronous UpDownCounter — one of the three instrument kinds the + // direct API unlocks — must be creatable and non-null. + auto upDown = meter->CreateInt64UpDownCounter("test_open_ledgers"); + ASSERT_TRUE(static_cast(upDown)); + + // Both increment and decrement paths must execute without error (inert on + // the noop instrument, but exercised here to prove both directions work). + upDown->Add(static_cast(1)); + upDown->Add(static_cast(-1)); +} + +// --------------------------------------------------------------- +// 2. Global-provider contract (the beast OTelCollector shim path). +// After a real SDK MeterProvider is registered via SetMeterProvider(), +// GetMeterProvider()->GetMeter(kMeterName, kMeterVersion) — the exact call +// the shim makes — must return a non-null meter, and an UpDownCounter built +// from it must be non-null and accept positive and negative Add(). +// --------------------------------------------------------------- +TEST(GetMeter, global_provider_meter_accepts_updown_counter) +{ + // Preserve and later restore the process-wide provider so this test does + // not leak state into other telemetry tests in the same binary. + auto const previous = metrics_api::Provider::GetMeterProvider(); + + // A views-less SDK MeterProvider with no reader is sufficient to prove the + // API contract: it hands out a real (non-noop) Meter that creates working + // instruments. No exporter/reader means no background threads or network. + std::shared_ptr const sdkProvider = + metrics_sdk::MeterProviderFactory::Create(); + metrics_api::Provider::SetMeterProvider( + opentelemetry::nostd::shared_ptr(sdkProvider)); + + // Fetch the meter exactly as the beast shim does: kMeterName + kMeterVersion + // from the global provider. + auto meter = metrics_api::Provider::GetMeterProvider()->GetMeter( + std::string(kMeterName), std::string(kMeterVersion)); + ASSERT_TRUE(static_cast(meter)); + + auto upDown = meter->CreateInt64UpDownCounter("test_in_flight_requests"); + ASSERT_TRUE(static_cast(upDown)); + + // Positive path: value goes up. + upDown->Add(static_cast(1)); + // Negative path: UpDownCounter permits decrement (unlike a plain Counter). + upDown->Add(static_cast(-1)); + + // Restore the previous global provider. + metrics_api::Provider::SetMeterProvider(previous); +} + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/MetricMacros.cpp b/src/tests/libxrpl/telemetry/MetricMacros.cpp new file mode 100644 index 00000000000..ddcd2af33c9 --- /dev/null +++ b/src/tests/libxrpl/telemetry/MetricMacros.cpp @@ -0,0 +1,338 @@ +/** + * @file MetricMacros.cpp + * Unit tests for the XRPL_METRIC_* call-site macros (MetricMacros.h). + * + * Compiled only when XRPL_ENABLE_TELEMETRY is defined -- the macros expand + * to no-ops otherwise and have nothing to test beyond "it compiles", which + * MetricsRegistry.cpp's own no-op build already proves. + * + * These tests exercise the macros against a bare SDK MeterProvider (no + * OTLP exporter, no network, no background thread), the same technique + * GetMeter.cpp uses. The macros are duck-typed: they only ever call + * app.getMetricsRegistry(), then isEnabled() and meter() on the result. + * The tests therefore drive them through a tiny FakeApp/FakeMetricsRegistry + * pair instead of the real telemetry::MetricsRegistry, whose enabled-path + * .cpp drags xrpld link dependencies (LedgerMaster, TxQ, NetworkOPs, ...) + * that the standalone xrpl_tests binary cannot satisfy -- the same reason + * MetricsRegistry.cpp is only compiled into this binary on the no-op path. + */ + +// cspell:ignore ISTOGRAM +// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's +// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. + +#ifdef XRPL_ENABLE_TELEMETRY + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace xrpl; + +namespace { + +/** + * Duck-typed stand-in for telemetry::MetricsRegistry. The macros only + * ever call isEnabled() and meter() on the registry pointer, so this + * minimal type is all they need -- and it links without pulling in the + * real MetricsRegistry.cpp's xrpld dependencies. + */ +class FakeMetricsRegistry +{ +public: + /** + * Sets the enable flag and meter the macros will observe. Tests call + * this once before exercising a macro, mimicking a started registry. + */ + void + configure(bool enabled, opentelemetry::nostd::shared_ptr meter) + { + enabled_ = enabled; + meter_ = std::move(meter); + } + + /** + * Number of times meter() has been consulted, so a test can assert the + * create-once (call_once) and disabled-gating behavior exactly. + */ + [[nodiscard]] int + meterCalls() const noexcept + { + return meterCalls_.load(std::memory_order_relaxed); + } + + [[nodiscard]] bool + isEnabled() const noexcept + { + return enabled_; + } + + [[nodiscard]] opentelemetry::nostd::shared_ptr + meter() const noexcept + { + meterCalls_.fetch_add(1, std::memory_order_relaxed); + return meter_; + } + +private: + /** + * Master enable flag the macro consults via isEnabled(). + */ + bool enabled_ = true; + + /** + * Meter handed to the macro; sourced from a bare SDK provider. + */ + opentelemetry::nostd::shared_ptr meter_; + + /** + * Counts meter() calls so tests can assert create-once / gating. + */ + mutable std::atomic meterCalls_{0}; +}; + +/** + * Duck-typed stand-in for ServiceRegistry. The macros only call + * getMetricsRegistry() on the app object, so this is all they need. + */ +class FakeApp +{ +public: + [[nodiscard]] FakeMetricsRegistry* + getMetricsRegistry() noexcept + { + return ®istry_; + } + + /** + * Direct handle to the underlying fake registry, for test setup and + * post-condition assertions. + */ + [[nodiscard]] FakeMetricsRegistry& + registry() noexcept + { + return registry_; + } + +private: + /** + * The macro-facing registry; tests configure it before use. + */ + FakeMetricsRegistry registry_; +}; + +/** + * Wraps a bare, exporter-less SDK MeterProvider so tests can build a + * real (non-noop) meter without touching the network. This is exactly + * the technique GetMeter.cpp's global_provider_meter_accepts_updown_counter + * test uses. + */ +class ScopedBareProvider +{ +public: + ScopedBareProvider() + { + // Hold the SDK provider in a std::shared_ptr first, then wrap it in a + // nostd::shared_ptr, exactly as GetMeter.cpp does. Constructing the + // nostd::shared_ptr directly from the factory's std::unique_ptr is + // ambiguous across its unique_ptr/shared_ptr overloads. + std::shared_ptr const sdkProvider = + opentelemetry::sdk::metrics::MeterProviderFactory::Create(); + previous_ = opentelemetry::metrics::Provider::GetMeterProvider(); + opentelemetry::metrics::Provider::SetMeterProvider( + opentelemetry::nostd::shared_ptr(sdkProvider)); + } + + ~ScopedBareProvider() + { + opentelemetry::metrics::Provider::SetMeterProvider(previous_); + } + +private: + /** + * Previous global provider, restored on scope exit. + */ + opentelemetry::nostd::shared_ptr previous_; +}; + +/** + * Fetches a real (non-noop) meter from whatever provider is globally + * installed -- inside a test this is the ScopedBareProvider's SDK provider. + */ +[[nodiscard]] opentelemetry::nostd::shared_ptr +bareMeter() +{ + return opentelemetry::metrics::Provider::GetMeterProvider()->GetMeter("xrpld_test", "1.0.0"); +} + +/** + * Builds a FakeApp wired to a real meter from the bare provider. + */ +void +wire(FakeApp& app, bool enabled) +{ + app.registry().configure(enabled, bareMeter()); +} + +} // namespace + +TEST(MetricMacros, counter_inc_creates_once_and_does_not_crash) +{ + ScopedBareProvider const bareProvider; + FakeApp app; + wire(app, /*enabled=*/true); + + // Call three times from the same call site; must not crash, and must + // not throw even though no real collector is listening. + for (int i = 0; i < 3; ++i) + { + XRPL_METRIC_COUNTER_INC( + app, "test_macro_counter_total", "Test counter for macro unit test"); + } + + // Create-once proof: std::call_once consults meter() exactly once across + // the three calls at this site, then reuses the cached instrument handle. + EXPECT_EQ(app.registry().meterCalls(), 1); +} + +TEST(MetricMacros, counter_inc_labeled_does_not_crash) +{ + ScopedBareProvider const bareProvider; + FakeApp app; + wire(app, /*enabled=*/true); + + XRPL_METRIC_COUNTER_INC_LABELED( + app, + "test_macro_labeled_counter_total", + "Test labeled counter for macro unit test", + {{"reason", std::string("unit_test")}}); + + // Instrument was created exactly once at this single call site. + EXPECT_EQ(app.registry().meterCalls(), 1); +} + +TEST(MetricMacros, histogram_record_does_not_crash) +{ + ScopedBareProvider const bareProvider; + FakeApp app; + wire(app, /*enabled=*/true); + + for (std::int64_t const v : {100, 250, 9999}) + { + XRPL_METRIC_HISTOGRAM_RECORD( + app, "test_macro_histogram_us", "Test histogram for macro unit test", v); + } + + // One call site, three records: histogram created once, then reused. + EXPECT_EQ(app.registry().meterCalls(), 1); +} + +TEST(MetricMacros, updown_add_accepts_positive_and_negative) +{ + ScopedBareProvider const bareProvider; + FakeApp app; + wire(app, /*enabled=*/true); + + // The whole point of UpDownCounter vs. Counter: negative Add() must not + // throw/crash/assert, unlike a plain Counter where a negative value + // violates the API contract. These are two distinct call sites (two + // source lines), so each lazily creates its own handle: meter() twice. + XRPL_METRIC_UPDOWN_ADD(app, "test_macro_updown_total", "Test updown for macro unit test", 1); + XRPL_METRIC_UPDOWN_ADD(app, "test_macro_updown_total", "Test updown for macro unit test", -1); + + EXPECT_EQ(app.registry().meterCalls(), 2); +} + +TEST(MetricMacros, updown_add_labeled_does_not_crash) +{ + ScopedBareProvider const bareProvider; + FakeApp app; + wire(app, /*enabled=*/true); + + XRPL_METRIC_UPDOWN_ADD_LABELED( + app, + "test_macro_updown_labeled_total", + "Test labeled updown for macro unit test", + -1, + {{"reason", std::string("unit_test")}}); + + EXPECT_EQ(app.registry().meterCalls(), 1); +} + +TEST(MetricMacros, observable_gauge_register_reports_current_value) +{ + ScopedBareProvider const bareProvider; + FakeApp app; + wire(app, /*enabled=*/true); + + // Own the state exactly as a real caller would (Use Case 5 in the + // design doc) -- the macro's callback reads through this atomic on + // every collection tick, it does not own the value itself. + std::atomic queueDepth{0}; + XRPL_METRIC_OBSERVABLE_GAUGE_REGISTER( + app, + "test_macro_observable_gauge", + "Test observable gauge for macro unit test", + [&queueDepth] { return queueDepth.load(); }); + + // There is no application-level read-back API (Use Case 4) -- this + // test can only prove registration doesn't crash and that meter() was + // consulted to create the observable instrument. It does NOT assert the + // observed value reaches Prometheus; that is Task 3b's docker-harness + // job, not this hermetic unit test. + queueDepth.store(42); + EXPECT_EQ(app.registry().meterCalls(), 1); +} + +TEST(MetricMacros, observable_counter_and_updown_register_do_not_crash) +{ + ScopedBareProvider const bareProvider; + FakeApp app; + wire(app, /*enabled=*/true); + + std::atomic total{0}; + XRPL_METRIC_OBSERVABLE_COUNTER_REGISTER( + app, + "test_macro_observable_counter_total", + "Test observable counter for macro unit test", + [&total] { return total.load(); }); + + std::atomic delta{0}; + XRPL_METRIC_OBSERVABLE_UPDOWN_REGISTER( + app, + "test_macro_observable_updown", + "Test observable updown for macro unit test", + [&delta] { return delta.load(); }); + + // Two observable registrations, each consulting meter() once. + EXPECT_EQ(app.registry().meterCalls(), 2); +} + +TEST(MetricMacros, disabled_registry_is_noop) +{ + ScopedBareProvider const bareProvider; + FakeApp app; + // enabled=false: the macro's isEnabled() gate short-circuits before it + // ever touches meter(), so nothing is created or recorded. + wire(app, /*enabled=*/false); + + XRPL_METRIC_COUNTER_INC( + app, "test_macro_disabled_counter_total", "Test counter for macro unit test (disabled)"); + + // Gating proof: meter() is never consulted when disabled. + EXPECT_EQ(app.registry().meterCalls(), 0); +} + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/MetricsRegistry.cpp b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp new file mode 100644 index 00000000000..7d4206ab002 --- /dev/null +++ b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp @@ -0,0 +1,628 @@ +/** + * GTest unit tests for MetricsRegistry. + * + * Two independent groups, split by what they can link: + * + * 1. sanitiseHandler() — the `handler` label sanitiser. Runs in **both** + * builds. sanitiseHandler() is a public static constexpr defined inline + * in the header, so it needs no part of MetricsRegistry.cpp on the link + * line. These tests therefore sit outside the guard below; putting them + * inside it would silently compile them out of the telemetry-enabled + * build, which is the build that actually exports the label. + * + * 2. The no-op / telemetry-disabled path — construction, start()/stop() + * lifecycle, and the synchronous record*() methods. Guarded, because + * when XRPL_ENABLE_TELEMETRY is defined MetricsRegistry.cpp is not + * compiled into this binary (see src/tests/libxrpl/CMakeLists.txt) and + * its out-of-line symbols are unresolvable here. + */ + +#include + +#include + +#include +#include +#include +#include + +namespace { + +using xrpl::telemetry::MetricsRegistry; + +/** + * Every job name reaching the JobQueue in non-test code that + * sanitiseHandler() must return unchanged, i.e. every one consisting solely + * of ASCII letters. + * + * How to re-derive this set (it is read off the source, not off docs): + * + * 1. Enumerate the four surfaces that put a name into the JobQueue, + * excluding `src/test/` and `src/tests/`: + * - `JobQueue::addJob` + * - `JobQueue::postCoro` (its `name` becomes `Coro::name_`, which + * `Coro::post()` hands to `addRefCountedJob`) + * - `LedgerMaster::newPFWork`, a thin `addJob` wrapper + * - each `TimeoutCounter` subclass's `.jobName =` designated + * initialiser, consumed by `TimeoutCounter::queueJob()`'s `addJob` + * `addRefCountedJob` is private and has only those callers, so the four + * surfaces close the set. + * 2. Resolve each `name` argument to the literal(s) it can hold. Most are + * literals written in place, but four call sites pass a variable and + * must be traced back: + * - `PeerImp.cpp` passes a local `std::string const name = + * isTrusted ? "ChkTrust" : "ChkUntrust";`, so one call site + * contributes *two* names. Neither literal appears at an `addJob` + * call site, which is why a grep of `addJob` alone misses them. + * - `LedgerMaster::newPFWork` forwards its own `char const* name` + * parameter; its three callers supply the three `PthFind*` literals. + * - `TimeoutCounter` forwards `queueJobParameter_.jobName`, set by a + * `.jobName =` designated initialiser in each of its five + * subclasses. + * - `Coro::post()` forwards `name_`, set from `postCoro`'s argument. + * 3. Keep the names that satisfy sanitiseHandler()'s rule — non-empty and + * all ASCII letters. Everything else belongs in kFoldToOtherHandlers. + * + * Note what step 3 excludes. Two call sites compose the name at runtime from + * a literal prefix and a ledger sequence: `"Pub" + std::to_string(seq)` and + * `"OB" + std::to_string(seq % 1000000000)`. The bare prefixes `"Pub"` and + * `"OB"` are all letters, but they are never what reaches the JobQueue — the + * *composed* form is, and it always carries digits. So they are absent here + * and their composed forms appear in kFoldToOtherHandlers instead. + * + * `JobQueue::makeLoadEvent` is deliberately out of scope: its `name` feeds + * `LoadEvent`/`LoadMonitor`, neither of which reaches MetricsRegistry, so + * names like `"cmd:" + method` never become a `handler` label value. + * + * Asserting on the real list is the point of the test: it proves the + * cardinality bound holds for the names actually in the binary, so a job + * added later whose name breaks the rule shows up as a failure here rather + * than as an unexplained `other` bucket on a dashboard. + */ +constexpr std::array kPassThroughHandlers = { + std::string_view{"AcceptLedger"}, // RCLConsensus.cpp + std::string_view{"AcqDone"}, // InboundLedger.cpp + std::string_view{"AdvanceLedger"}, // LedgerMaster.cpp + std::string_view{"ChkTrust"}, // PeerImp.cpp (local, ternary) + std::string_view{"ChkUntrust"}, // PeerImp.cpp (local, ternary) + std::string_view{"ComplAcquire"}, // TransactionAcquire.cpp + std::string_view{"DoTxs"}, // PeerImp.cpp + std::string_view{"GotFetchPack"}, // LedgerMaster.cpp + std::string_view{"GotStaleData"}, // InboundLedgers.cpp + std::string_view{"HandleHaveTxs"}, // PeerImp.cpp + std::string_view{"HistTxStream"}, // NetworkOPs.cpp + std::string_view{"InboundLedger"}, // InboundLedger.cpp (.jobName) + std::string_view{"LedReplDelta"}, // LedgerDeltaAcquire.cpp (.jobName) + std::string_view{"LedReplTask"}, // LedgerReplayTask.cpp (.jobName) + std::string_view{"MakeFetchPack"}, // PeerImp.cpp + std::string_view{"NObjStore"}, // NodeStoreScheduler.cpp + std::string_view{"NetCluster"}, // NetworkOPs.cpp + std::string_view{"NetHeart"}, // NetworkOPs.cpp + std::string_view{"OnLedBuilt"}, // LedgerDeltaAcquire.cpp + std::string_view{"ProcessLData"}, // InboundLedgers.cpp + std::string_view{"PthFindNewLed"}, // LedgerMaster.cpp (newPFWork) + std::string_view{"PthFindNewReq"}, // LedgerMaster.cpp (newPFWork) + std::string_view{"PthFindOBDB"}, // LedgerMaster.cpp (newPFWork) + std::string_view{"PubCons"}, // NetworkOPs.cpp + std::string_view{"PubFee"}, // NetworkOPs.cpp + std::string_view{"RPCSubSendThr"}, // RPCSub.cpp + std::string_view{"RcvCheckTx"}, // PeerImp.cpp + std::string_view{"RcvGetLedger"}, // PeerImp.cpp + std::string_view{"RcvGetObjByHash"}, // PeerImp.cpp + std::string_view{"RcvManifests"}, // PeerImp.cpp + std::string_view{"RcvPeerData"}, // PeerImp.cpp + std::string_view{"RcvProofPReq"}, // PeerImp.cpp + std::string_view{"RcvReplDReq"}, // PeerImp.cpp + std::string_view{"SkipListAcq"}, // SkipListAcquire.cpp (.jobName) + std::string_view{"SubmitTxn"}, // NetworkOPs.cpp + std::string_view{"TryFill"}, // LedgerMaster.cpp + std::string_view{"TxAcq"}, // TransactionAcquire.cpp (.jobName) + std::string_view{"TxBatchAsync"}, // NetworkOPs.cpp + std::string_view{"TxBatchSync"}, // NetworkOPs.cpp + std::string_view{"TxsToTxn"}, // ConsensusTransSetSF.cpp + std::string_view{"WAL"}, // SociDB.cpp + std::string_view{"checkPropose"}, // PeerImp.cpp (lowercase start) + std::string_view{"sweep"}, // Application.cpp (all lowercase) +}; + +/** + * Names that must fold to kHandlerOther. + * + * The first two are the composed forms of the only two dynamically built job + * names in the tree: `"Pub" + std::to_string(ledger->seq())` + * (LedgerPersistence.cpp) and + * `"OB" + std::to_string(ledger->seq() % 1000000000)` (OrderBookDBImpl.cpp). + * They are the reason the sanitiser exists — used raw they would mint a + * Prometheus series per ledger — so realistic sequence values are used + * rather than short placeholders. The sequence is unbounded at the `"Pub"` + * site and masked to nine digits at the `"OB"` site, but a digit appears + * either way (even `seq == 0` gives `"Pub0"`), so no reachable input at + * either site can produce an all-letter name. + * + * The next five are static literals that already fail the rule today, so + * the fallback is exercised by real code and not only by synthetic input. + * + * The remainder are the edge cases: empty, and one entry per disallowed + * character class (space, digit, underscore, hyphen, non-ASCII byte). The + * non-ASCII entry is UTF-8 'e-acute'; on a signed-char platform its lead + * byte is negative, which the explicit ASCII range check rejects where a + * locale-sensitive std::isalpha might not. + */ +constexpr std::array kFoldToOtherHandlers = { + std::string_view{"Pub97531234"}, // dynamic: "Pub" + ledger seq + std::string_view{"OB123456789"}, // dynamic: "OB" + ledger seq % 1e9 + std::string_view{"GetConsL1"}, // static, digit (RCLConsensus.cpp) + std::string_view{"GetConsL2"}, // static, digit (RCLValidations.cpp) + std::string_view{"gRPC-Client"}, // static, hyphen (GRPCServer.cpp) + std::string_view{"RPC-Client"}, // static, hyphen (ServerHandler.cpp) + std::string_view{"WS-Client"}, // static, hyphen (ServerHandler.cpp) + std::string_view{""}, // empty + std::string_view{"Rcv Ledger"}, // space + std::string_view{"Handler7"}, // digit + std::string_view{"Rcv_Ledger"}, // underscore + std::string_view{"Rcv-Ledger"}, // hyphen + std::string_view{"caf\xC3\xA9"}, // non-ASCII byte (UTF-8 e-acute) +}; + +/** + * True when sanitiseHandler() returns each pass-through name unchanged. + * + * consteval so a regression is a compile error rather than a test failure: + * the sanitiser is constexpr precisely so this bound can be checked without + * running anything. + */ +consteval bool +allPassThroughUnchanged() +{ + for (auto const name : kPassThroughHandlers) + { + if (MetricsRegistry::sanitiseHandler(name) != name) + return false; + } + return true; +} + +/** + * True when sanitiseHandler() maps every listed name to kHandlerOther. + */ +consteval bool +allFoldToOther() +{ + for (auto const name : kFoldToOtherHandlers) + { + if (MetricsRegistry::sanitiseHandler(name) != MetricsRegistry::kHandlerOther) + return false; + } + return true; +} + +// Compile-time guarantees. Duplicated at runtime below so a failure names +// the offending input instead of only pointing at the assertion. +static_assert(allPassThroughUnchanged()); +static_assert(allFoldToOther()); + +// The verified size of the pass-through set as of this branch: 43 all-letter +// job-name literals. Pinned so that adding or removing a job name without +// revisiting the label-cardinality budget fails the build here. +static_assert(kPassThroughHandlers.size() == 43); + +/** + * Total distinct `handler` label values reachable from the inputs above: + * one per pass-through name plus the single shared kHandlerOther bucket. + * This is the number the Prometheus cardinality budget is sized against. + */ +constexpr std::size_t kExpectedHandlerDomain = kPassThroughHandlers.size() + 1; +static_assert(kExpectedHandlerDomain == 44); + +} // namespace + +TEST(MetricsRegistrySanitiseHandler, static_job_names_pass_through_unchanged) +{ + // Every all-letter job name in the tree survives sanitisation, so the + // `handler` label keeps its attribution value for real producers. + for (auto const name : kPassThroughHandlers) + { + EXPECT_EQ(MetricsRegistry::sanitiseHandler(name), name) + << "job name should pass through unchanged: " << name; + // Cause, not just state: it passed because it is not the fallback. + EXPECT_NE(MetricsRegistry::sanitiseHandler(name), MetricsRegistry::kHandlerOther) + << "job name wrongly folded to the fallback: " << name; + } +} + +TEST(MetricsRegistrySanitiseHandler, dynamic_and_non_letter_names_fold_to_other) +{ + // Negative path: everything that is not an all-letter name collapses + // into exactly one bucket, which is what bounds the label domain. + for (auto const name : kFoldToOtherHandlers) + { + EXPECT_EQ(MetricsRegistry::sanitiseHandler(name), MetricsRegistry::kHandlerOther) + << "name should fold to the fallback: " << name; + } +} + +TEST(MetricsRegistrySanitiseHandler, empty_name_folds_to_other) +{ + // Called out separately because it is the one case the all-letter scan + // cannot catch: std::ranges::all_of() is vacuously true on an empty + // range, so the sanitiser needs its own emptiness check. + EXPECT_EQ(MetricsRegistry::sanitiseHandler(std::string_view{}), MetricsRegistry::kHandlerOther); + EXPECT_EQ(MetricsRegistry::sanitiseHandler(""), MetricsRegistry::kHandlerOther); +} + +TEST(MetricsRegistrySanitiseHandler, fallback_value_is_the_shared_constant) +{ + // The fallback must be the constant the dashboards and the reference doc + // are written against, not merely some non-empty string. + EXPECT_EQ(MetricsRegistry::kHandlerOther, std::string_view{"other"}); + EXPECT_EQ(MetricsRegistry::kHandlerOther.size(), 5u); + + // "other" is itself all letters, so sanitising it is idempotent -- a + // handler genuinely named "other" is indistinguishable from the bucket. + EXPECT_EQ( + MetricsRegistry::sanitiseHandler(MetricsRegistry::kHandlerOther), + MetricsRegistry::kHandlerOther); +} + +TEST(MetricsRegistrySanitiseHandler, output_domain_is_exactly_44_values) +{ + // The cardinality bound itself: over every input above -- 43 real job + // names, 2 dynamic names, 5 non-conforming static names and 6 edge + // cases -- the sanitiser can emit only 44 distinct label values (43 + // names plus the single "other" bucket). + std::set domain; + for (auto const name : kPassThroughHandlers) + domain.insert(MetricsRegistry::sanitiseHandler(name)); + for (auto const name : kFoldToOtherHandlers) + domain.insert(MetricsRegistry::sanitiseHandler(name)); + + EXPECT_EQ(domain.size(), kExpectedHandlerDomain); + EXPECT_EQ(domain.size(), 44u); + + // State plus cause: the domain is the pass-through names and nothing + // else besides the one fallback bucket. + EXPECT_TRUE(domain.contains(MetricsRegistry::kHandlerOther)); + EXPECT_EQ(domain.size() - 1, kPassThroughHandlers.size()); + for (auto const name : kPassThroughHandlers) + EXPECT_TRUE(domain.contains(name)) << "missing from domain: " << name; +} + +// When telemetry is globally enabled, MetricsRegistry.cpp requires xrpld +// link dependencies we cannot satisfy in a standalone GTest binary. +#ifndef XRPL_ENABLE_TELEMETRY + +#include +#include +#include +#include + +#include + +#include +#include +#include + +using namespace xrpl; + +namespace { + +/** + * Minimal mock ServiceRegistry for MetricsRegistry testing. + * + * Only the getMetricsRegistry() call is used in the tests; other methods + * are not invoked because the registry is disabled (enabled=false) so no + * gauge callbacks execute. + * + * All pure virtual methods throw to catch accidental calls during tests. + */ +class MockServiceRegistry : public ServiceRegistry +{ + [[noreturn]] static void + throwUnimplemented() + { + throw std::logic_error("MockServiceRegistry: method not implemented"); + } + +public: + // ServiceRegistry interface — stubs that should never be called. + CollectorManager& + getCollectorManager() override + { + throwUnimplemented(); + } + Family& + getNodeFamily() override + { + throwUnimplemented(); + } + TimeKeeper& + getTimeKeeper() override + { + throwUnimplemented(); + } + JobQueue& + getJobQueue() override + { + throwUnimplemented(); + } + NodeCache& + getTempNodeCache() override + { + throwUnimplemented(); + } + CachedSLEs& + getCachedSLEs() override + { + throwUnimplemented(); + } + NetworkIDService& + getNetworkIDService() override + { + throwUnimplemented(); + } + AmendmentTable& + getAmendmentTable() override + { + throwUnimplemented(); + } + HashRouter& + getHashRouter() override + { + throwUnimplemented(); + } + LoadFeeTrack& + getFeeTrack() override + { + throwUnimplemented(); + } + LoadManager& + getLoadManager() override + { + throwUnimplemented(); + } + RCLValidations& + getValidations() override + { + throwUnimplemented(); + } + ValidatorList& + getValidators() override + { + throwUnimplemented(); + } + ValidatorSite& + getValidatorSites() override + { + throwUnimplemented(); + } + ManifestCache& + getValidatorManifests() override + { + throwUnimplemented(); + } + ManifestCache& + getPublisherManifests() override + { + throwUnimplemented(); + } + Overlay& + getOverlay() override + { + throwUnimplemented(); + } + Cluster& + getCluster() override + { + throwUnimplemented(); + } + PeerReservationTable& + getPeerReservations() override + { + throwUnimplemented(); + } + Resource::Manager& + getResourceManager() override + { + throwUnimplemented(); + } + NodeStore::Database& + getNodeStore() override + { + throwUnimplemented(); + } + SHAMapStore& + getSHAMapStore() override + { + throwUnimplemented(); + } + RelationalDatabase& + getRelationalDatabase() override + { + throwUnimplemented(); + } + InboundLedgers& + getInboundLedgers() override + { + throwUnimplemented(); + } + InboundTransactions& + getInboundTransactions() override + { + throwUnimplemented(); + } + TaggedCache& + getAcceptedLedgerCache() override + { + throwUnimplemented(); + } + LedgerMaster& + getLedgerMaster() override + { + throwUnimplemented(); + } + LedgerCleaner& + getLedgerCleaner() override + { + throwUnimplemented(); + } + LedgerReplayer& + getLedgerReplayer() override + { + throwUnimplemented(); + } + PendingSaves& + getPendingSaves() override + { + throwUnimplemented(); + } + [[nodiscard]] OpenLedger& + getOpenLedger() override + { + throwUnimplemented(); + } + [[nodiscard]] OpenLedger const& + getOpenLedger() const override + { + throwUnimplemented(); + } + NetworkOPs& + getOPs() override + { + throwUnimplemented(); + } + OrderBookDB& + getOrderBookDB() override + { + throwUnimplemented(); + } + TransactionMaster& + getMasterTransaction() override + { + throwUnimplemented(); + } + TxQ& + getTxQ() override + { + throwUnimplemented(); + } + PathRequestManager& + getPathRequestManager() override + { + throwUnimplemented(); + } + ServerHandler& + getServerHandler() override + { + throwUnimplemented(); + } + perf::PerfLog& + getPerfLog() override + { + throwUnimplemented(); + } + telemetry::Telemetry& + getTelemetry() override + { + throwUnimplemented(); + } + telemetry::MetricsRegistry* + getMetricsRegistry() override + { + return nullptr; + } + [[nodiscard]] bool + isStopping() const override + { + return false; + } + beast::Journal + getJournal(std::string const&) override + { + return beast::Journal(beast::Journal::getNullSink()); + } + boost::asio::io_context& + getIOContext() override + { + throwUnimplemented(); + } + Logs& + getLogs() override + { + throwUnimplemented(); + } + [[nodiscard]] std::optional const& + getTrapTxID() const override + { + static std::optional const empty; + return empty; + } + DatabaseCon& + getWalletDB() override + { + throwUnimplemented(); + } + Application& + getApp() override + { + throwUnimplemented(); + } +}; + +/** + * Test fixture that provides a MockServiceRegistry and null Journal. + */ +class MetricsRegistryTest : public ::testing::Test +{ +protected: + MockServiceRegistry mockApp_; + beast::Journal j_{beast::Journal::getNullSink()}; +}; + +} // namespace + +TEST_F(MetricsRegistryTest, disabled_construction) +{ + // Construct with enabled=false; should be a no-op. + telemetry::MetricsRegistry const registry(false, mockApp_, j_); + EXPECT_FALSE(registry.isEnabled()); +} + +TEST_F(MetricsRegistryTest, disabled_start_stop) +{ + telemetry::MetricsRegistry registry(false, mockApp_, j_); + + // start() and stop() should be no-ops when disabled. + registry.start("http://localhost:4318/v1/metrics"); + registry.stop(); + + // Double stop should be safe. + registry.stop(); +} + +TEST_F(MetricsRegistryTest, disabled_recording_methods) +{ + telemetry::MetricsRegistry registry(false, mockApp_, j_); + registry.start("http://localhost:4318/v1/metrics"); + + // All recording methods should be no-ops (not crash). + registry.recordRpcStarted("server_info"); + registry.recordRpcFinished("server_info", 1000); + registry.recordRpcErrored("ledger", 500); + registry.recordJobQueued("ledgerData", "ProcessLData"); + registry.recordJobStarted("ledgerData", "ProcessLData", 200); + registry.recordJobFinished("ledgerData", "ProcessLData", 3000); + + registry.stop(); +} + +TEST_F(MetricsRegistryTest, destructor_calls_stop) +{ + { + // Let the destructor handle cleanup. + telemetry::MetricsRegistry registry(false, mockApp_, j_); + registry.start("http://localhost:4318/v1/metrics"); + } + // If we get here without crash, the destructor handled stop. +} + +#endif // !XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/Redaction.cpp b/src/tests/libxrpl/telemetry/Redaction.cpp new file mode 100644 index 00000000000..2ef6ed377b7 --- /dev/null +++ b/src/tests/libxrpl/telemetry/Redaction.cpp @@ -0,0 +1,44 @@ +#include + +#include + +#include +#include +#include + +using namespace xrpl; +using namespace xrpl::telemetry; + +// Empty input must produce empty output (edge / negative path). +TEST(Redaction, empty_input_returns_empty) +{ + EXPECT_EQ(redactAccount(""), ""); +} + +// Success path: assert the EXACT expected token. The value is the first +// 16 chars of sha512Half(addr) rendered as lowercase hex, computed once +// and hard-coded here so any change to the hashing recipe is caught. +TEST(Redaction, known_account_exact_hash) +{ + EXPECT_EQ(redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"), "a513895f49311f54"); +} + +// Structural invariants: length, lowercase-hex alphabet, and stability. +TEST(Redaction, token_is_16_lowercase_hex_and_stable) +{ + auto const token = redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"); + EXPECT_EQ(token.size(), 16u); + EXPECT_TRUE(std::ranges::all_of(token, [](unsigned char c) { + return std::isdigit(c) || (c >= 'a' && c <= 'f'); + })); + // Deterministic: hashing the same input twice yields the same token. + EXPECT_EQ(token, redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh")); +} + +// Distinct inputs must produce distinct tokens (no accidental constant). +TEST(Redaction, distinct_inputs_distinct_tokens) +{ + EXPECT_NE( + redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"), + redactAccount("rDifferentAccount1234567890abcdefgh")); +} diff --git a/src/tests/libxrpl/telemetry/SpanGuardFactory.cpp b/src/tests/libxrpl/telemetry/SpanGuardFactory.cpp new file mode 100644 index 00000000000..75e5804bbb0 --- /dev/null +++ b/src/tests/libxrpl/telemetry/SpanGuardFactory.cpp @@ -0,0 +1,114 @@ +#include +#include + +#include + +#include +#include +#include +#include +#include + +using namespace xrpl; +using namespace xrpl::telemetry; + +TEST(SpanGuardFactory, null_guard_methods_are_safe) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "nonexistent"); + EXPECT_FALSE(span); + + span.setAttribute("key", "value"); + span.setAttribute("int_key", static_cast(42)); + span.setAttribute("bool_key", true); + span.setOk(); + span.setError("test"); + span.addEvent("event"); +} + +TEST(SpanGuardFactory, category_span_returns_null_when_disabled) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "test"); + EXPECT_FALSE(span); + + // Attribute keys use the underscore convention for span attributes (the + // dotted xrpl.. form is reserved for resource attributes). The + // canonical constants live in the xrpld-level *SpanNames.h headers, which a + // libxrpl test cannot include, so the keys are written as literals here. + span.setAttribute("command", "test"); + span.setAttribute("rpc_status", "success"); +} + +TEST(SpanGuardFactory, child_span_null_when_no_parent) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "parent"); + auto child = span.childSpan("child.test"); + EXPECT_FALSE(child); +} + +TEST(SpanGuardFactory, linked_span_null_when_no_context) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "source"); + auto linked = span.linkedSpan("linked.test"); + EXPECT_FALSE(linked); +} + +TEST(SpanGuardFactory, span_context_returns_invalid_on_null) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "ctx"); + auto ctx = span.spanContext(); + EXPECT_FALSE(ctx.isValid()); +} + +TEST(SpanGuardFactory, move_construction_transfers_ownership) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "move"); + auto moved = std::move(span); + EXPECT_FALSE(span); // NOLINT(bugprone-use-after-move,hicpp-invalid-access-moved) + moved.setAttribute("key", "value"); +} + +TEST(SpanGuardFactory, record_exception_safe_on_null) +{ + auto span = SpanGuard::span(TraceCategory::Rpc, "rpc.command", "test"); + try + { + throw std::runtime_error("test error"); + } + catch (std::exception const& e) + { + span.recordException(e); + } +} + +TEST(SpanGuardFactory, discard_safe_on_null) +{ + auto span = SpanGuard::span(TraceCategory::Transactions, "tx", "process"); + span.discard(); + EXPECT_FALSE(span); +} + +TEST(SpanGuardFactory, consensus_close_time_attributes) +{ + // Verify the consensus attribute pattern compiles and doesn't crash with a + // null SpanGuard. Attribute keys/values use the underscore convention; the + // canonical consensus::span constants are defined in the xrpld-level + // ConsensusSpanNames.h, which a libxrpl test cannot include, so the keys are + // written as literals here. + { + auto span = telemetry::SpanGuard::span( + telemetry::TraceCategory::Consensus, telemetry::seg::consensus, "accept.apply"); + span.setAttribute("ledger_seq", static_cast(42)); + span.setAttribute("close_time", static_cast(780000000)); + span.setAttribute("close_time_correct", true); + span.setAttribute("close_resolution_ms", static_cast(30000)); + span.setAttribute("consensus_state", std::string("finished")); + span.setAttribute("proposing", true); + span.setAttribute("round_time_ms", static_cast(3500)); + } + { + auto span = telemetry::SpanGuard::span( + telemetry::TraceCategory::Consensus, telemetry::seg::consensus, "accept.apply"); + span.setAttribute("close_time_correct", false); + span.setAttribute("consensus_state", std::string("moved_on")); + } +} diff --git a/src/tests/libxrpl/telemetry/SpanGuardScope.cpp b/src/tests/libxrpl/telemetry/SpanGuardScope.cpp new file mode 100644 index 00000000000..8795b4f6147 --- /dev/null +++ b/src/tests/libxrpl/telemetry/SpanGuardScope.cpp @@ -0,0 +1,739 @@ +// Tests for SpanGuard, ScopedSpanGuard and DeterministicIdGenerator. +// +// These verify the span-guard split and the deterministic-root fix at the +// trace level using an in-memory span exporter: +// - SpanGuard is unscoped and thread-free: it owns only the span, never the +// OTel thread-local context stack, so it may be moved to and ended on any +// thread. SpanGuard::freshRoot() starts a brand-new trace root, ignoring +// the ambient active span. +// - ScopedSpanGuard is scoped and store-bound: it also pushes an OTel Scope +// so the span is the ambient parent on the constructing context store (the +// thread's own store, or a coroutine's store). Its `operator SpanGuard() &&` +// pops that Scope eagerly on the origin store and yields a thread-free +// SpanGuard; destroying it while a different store is active trips an +// owner-store assertion. +// - DeterministicIdGenerator (installed by the test TracerProvider) mints a +// caller-pinned trace_id for a forced-root span. PendingTraceId pins the id +// for one root span; an ambient child under a live parent never adopts it. +// +// The whole file is telemetry-only: when XRPL_ENABLE_TELEMETRY is not defined +// SpanGuard is a no-op stub and the OpenTelemetry SDK headers are unavailable, +// so the translation unit compiles empty. + +#ifdef XRPL_ENABLE_TELEMETRY + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::telemetry { +namespace { + +namespace otel_sdk_trace = opentelemetry::sdk::trace; +namespace otel_memory = opentelemetry::exporter::memory; + +/** + * In-memory Telemetry backing for SpanGuard scope tests. + * + * Reports every trace category as enabled and creates spans through an SDK + * TracerProvider whose SimpleSpanProcessor forwards ended spans to an + * InMemorySpanExporter, so a test can read the exact exported SpanData + * (trace id, span id, parent id, name). The provider is built with a + * DeterministicIdGenerator so PendingTraceId can pin the trace_id of a + * forced-root span. + * + * Inheritance: + * + * +-----------+ + * | Telemetry | (abstract interface) + * +-----+-----+ + * | + * +-----+---------+ + * | TestTelemetry | in-memory exporter pipeline + * +---------------+ + * + * @note Test-only. Install with Telemetry::setInstance() and clear it in + * teardown. The exporter buffer is drained by InMemorySpanData::GetSpans(), + * so each GetSpans() call returns only spans exported since the previous one. + */ +class TestTelemetry : public Telemetry +{ +public: + /** + * Build the SDK export pipeline and keep the exporter's span buffer. + */ + TestTelemetry() + { + // Factory populates spanData_ with the exporter's shared buffer. + auto exporter = otel_memory::InMemorySpanExporterFactory::Create(spanData_); + auto processor = otel_sdk_trace::SimpleSpanProcessorFactory::Create(std::move(exporter)); + // Install the DeterministicIdGenerator (4-arg overload) so a + // PendingTraceId can pin a forced-root span's trace_id in tests. + provider_ = otel_sdk_trace::TracerProviderFactory::Create( + std::move(processor), + opentelemetry::sdk::resource::Resource::Create({}), + otel_sdk_trace::AlwaysOnSamplerFactory::Create(), + std::make_unique()); + } + + /** + * @return The exporter's span buffer (drained by GetSpans()). + */ + [[nodiscard]] std::shared_ptr + spanData() const + { + return spanData_; + } + + void + start() override + { + } + void + stop() override + { + } + + [[nodiscard]] bool + isEnabled() const override + { + return true; + } + [[nodiscard]] bool + shouldTraceTransactions() const override + { + return true; + } + [[nodiscard]] bool + shouldTraceConsensus() const override + { + return true; + } + [[nodiscard]] bool + shouldTraceRpc() const override + { + return true; + } + [[nodiscard]] bool + shouldTracePeer() const override + { + return true; + } + [[nodiscard]] bool + shouldTraceLedger() const override + { + return true; + } + + /** + * @return A fixed strategy label; the scope tests do not exercise + * deterministic trace-id correlation, so any stable value works. + */ + [[nodiscard]] std::string const& + getConsensusTraceStrategy() const override + { + static std::string const kStrategy{"none"}; + return kStrategy; + } + + opentelemetry::nostd::shared_ptr + getTracer(std::string_view name) override + { + return provider_->GetTracer(std::string(name)); + } + + /** + * @return A meter from a noop provider; the scope tests exercise only + * tracing, so metrics instruments are inert. + */ + opentelemetry::nostd::shared_ptr + getMeter(std::string_view name) override + { + static auto noopProvider = + opentelemetry::nostd::shared_ptr( + new opentelemetry::metrics::NoopMeterProvider()); + return noopProvider->GetMeter(std::string(name), std::string(kMeterVersion)); + } + + opentelemetry::nostd::shared_ptr + startSpan(std::string_view name, opentelemetry::trace::SpanKind kind) override + { + opentelemetry::trace::StartSpanOptions opts; + opts.kind = kind; + return getTracer(kTracerName)->StartSpan(std::string(name), opts); + } + + opentelemetry::nostd::shared_ptr + startSpan( + std::string_view name, + opentelemetry::context::Context const& parentContext, + opentelemetry::trace::SpanKind kind) override + { + opentelemetry::trace::StartSpanOptions opts; + opts.kind = kind; + opts.parent = parentContext; + return getTracer(kTracerName)->StartSpan(std::string(name), opts); + } + +private: + /** + * SDK provider owning the export pipeline. + */ + std::shared_ptr provider_; + + /** + * Shared buffer that receives ended spans from the exporter. + */ + std::shared_ptr spanData_; +}; + +/** + * @return The span's name as a std::string for equality checks. + */ +std::string +nameOf(otel_sdk_trace::SpanData const& span) +{ + auto view = span.GetName(); + return std::string(view.data(), view.size()); +} + +/** + * @return Pointer to the first exported span with the given name, or null. + */ +otel_sdk_trace::SpanData* +findSpan(std::vector> const& spans, std::string_view name) +{ + for (auto const& span : spans) + { + if (nameOf(*span) == name) + return span.get(); + } + return nullptr; +} + +/** + * @return Number of exported spans with the given name. + */ +std::size_t +countSpans( + std::vector> const& spans, + std::string_view name) +{ + std::size_t count = 0; + for (auto const& span : spans) + { + if (nameOf(*span) == name) + ++count; + } + return count; +} + +/** + * Build the 16-byte deterministic trace_id used by the generator tests + * (bytes 1..16). Kept out of line so every generator test pins the same id. + * @return The fixed 16-byte trace_id {1, 2, ..., 16}. + */ +std::array +makeTraceIdBytes() +{ + std::array h{}; + for (int i = 0; i < 16; ++i) + h[i] = static_cast(i + 1); + return h; +} + +/** + * Installs a TestTelemetry as the global instance for each test and clears it + * afterwards so the singleton never dangles between cases. + * + * The fixture also installs a CoroAwareContextStorage as the process-global + * OTel runtime-context storage. That storage backs the ambient-context stack + * with an xrpl::LocalValue, which is what makes a ScopedSpanGuard's Scope live + * in the ACTIVE store (thread or coroutine) rather than a plain thread_local. + * The store-swap and activate() tests depend on this to observe the ambient + * context move with the store. + * + * @note SetRuntimeContextStorage is process-global. Re-installing it in every + * SetUp is idempotent (last writer wins) and safe because GTests run serially; + * a fresh storage per test keeps the coro/thread context stacks isolated. + */ +class SpanGuardScopeTest : public ::testing::Test +{ +protected: + void + SetUp() override + { + telemetry_ = std::make_unique(); + Telemetry::setInstance(telemetry_.get()); + + // Back the OTel ambient context with an xrpl::LocalValue store so a + // scope follows the active store, not a bare thread_local. Installed + // before any span is created in the test body (SDK requirement). + storage_ = opentelemetry::nostd::shared_ptr( + new CoroAwareContextStorage()); + opentelemetry::context::RuntimeContext::SetRuntimeContextStorage(storage_); + } + + void + TearDown() override + { + Telemetry::setInstance(nullptr); + telemetry_.reset(); + } + + /** + * @return The exporter's span buffer for the active TestTelemetry. + */ + [[nodiscard]] std::shared_ptr + spanData() const + { + return telemetry_->spanData(); + } + + /** + * The in-memory Telemetry installed for the duration of a test. + */ + std::unique_ptr telemetry_; + + /** + * The coro-aware runtime-context storage installed for the test. Kept + * alive by the fixture so the process-global storage pointer stays valid + * for the test body; replaced each SetUp (see class note). + */ + opentelemetry::nostd::shared_ptr storage_; +}; + +// freshRoot() must ignore the ambient active span and start a brand-new trace. +TEST_F(SpanGuardScopeTest, spanGuard_freshRoot_is_true_root_ignoring_ambient) +{ + { + // Ambient span becomes the active span on this thread. + ScopedSpanGuard const ambient(TraceCategory::Rpc, "rpc", "command"); + ASSERT_TRUE(static_cast(ambient)); + + // freshRoot must NOT inherit the ambient span as its parent. + auto r = SpanGuard::freshRoot(TraceCategory::Peer, "peer", "validation.receive"); + ASSERT_TRUE(static_cast(r)); + } // r ends first, then ambient's scope pops and ambient span ends. + + auto spans = spanData()->GetSpans(); + auto* ambient = findSpan(spans, "rpc.command"); + auto* root = findSpan(spans, "peer.validation.receive"); + ASSERT_NE(ambient, nullptr); + ASSERT_NE(root, nullptr); + + // The ambient span is itself a root (first span, no parent). + EXPECT_FALSE(ambient->GetParentSpanId().IsValid()); + EXPECT_TRUE(ambient->GetTraceId().IsValid()); + + // The freshRoot span has NO parent and lives in a DIFFERENT trace. + EXPECT_FALSE(root->GetParentSpanId().IsValid()); + EXPECT_TRUE(root->GetTraceId().IsValid()); + EXPECT_NE(root->GetTraceId(), ambient->GetTraceId()); +} + +// A ScopedSpanGuard is the ambient active span on its thread the moment it is +// constructed: a child created while it is alive parents to its span. +TEST_F(SpanGuardScopeTest, scopedGuard_is_ambient_on_construct) +{ + opentelemetry::trace::SpanId activeId; + { + ScopedSpanGuard const s(TraceCategory::Rpc, "rpc", "process"); + ASSERT_TRUE(static_cast(s)); + + // While s is alive it is the active span on this thread's context. + auto active = + opentelemetry::trace::GetSpan(opentelemetry::context::RuntimeContext::GetCurrent()) + ->GetContext(); + ASSERT_TRUE(active.IsValid()); + activeId = active.span_id(); + + // A child created here must parent to s's span, proving s is ambient. + auto child = s.childSpan("rpc.dispatch"); + ASSERT_TRUE(static_cast(child)); + } // child ends first, then s pops its scope and ends. + + auto spans = spanData()->GetSpans(); + auto* parent = findSpan(spans, "rpc.process"); + auto* child = findSpan(spans, "rpc.dispatch"); + ASSERT_NE(parent, nullptr); + ASSERT_NE(child, nullptr); + + // The active span observed while s was alive WAS s's span. + EXPECT_TRUE(activeId.IsValid()); + EXPECT_EQ(parent->GetSpanId(), activeId); + + // The child nested under s: same trace, parent = s's span. + EXPECT_EQ(child->GetParentSpanId(), parent->GetSpanId()); + EXPECT_EQ(child->GetTraceId(), parent->GetTraceId()); +} + +// operator SpanGuard() && pops the Scope eagerly on the origin thread, so the +// span is no longer ambient here and the resulting thread-free guard can be +// ended on a worker thread without corrupting this thread's context stack. +TEST_F(SpanGuardScopeTest, scopedGuard_conversion_pops_scope_on_this_thread) +{ + { + ScopedSpanGuard s(TraceCategory::Ledger, "ledger", "build"); + ASSERT_TRUE(static_cast(s)); + + // Convert to a bare SpanGuard: the scope is popped here, on this thread. + SpanGuard bare = std::move(s); + ASSERT_TRUE(static_cast(bare)); + + // The scope is gone: no span is active on this thread now. + auto active = + opentelemetry::trace::GetSpan(opentelemetry::context::RuntimeContext::GetCurrent()) + ->GetContext(); + EXPECT_FALSE(active.IsValid()); + + // A new ambient span here is a fresh root, NOT nested under build. + { + ScopedSpanGuard const after(TraceCategory::Rpc, "rpc", "command"); + ASSERT_TRUE(static_cast(after)); + } + + // End the thread-free guard on a worker thread -- no crash. + std::thread worker([g = std::move(bare)]() mutable {}); + worker.join(); + } + + auto spans = spanData()->GetSpans(); + auto* build = findSpan(spans, "ledger.build"); + auto* after = findSpan(spans, "rpc.command"); + ASSERT_NE(build, nullptr); + ASSERT_NE(after, nullptr); + + // The span was exported exactly once, by the worker thread. + EXPECT_EQ(countSpans(spans, "ledger.build"), 1u); + + // 'after' did NOT nest under build: fresh root, different trace. + EXPECT_FALSE(after->GetParentSpanId().IsValid()); + EXPECT_NE(after->GetTraceId(), build->GetTraceId()); +} + +// The SpanGuard produced by the conversion ends the span exactly once: the +// moved-from ScopedSpanGuard must not re-end it on destruction. +TEST_F(SpanGuardScopeTest, scopedGuard_conversion_result_ends_span_once) +{ + { + ScopedSpanGuard scoped(TraceCategory::Ledger, "ledger", "build"); + ASSERT_TRUE(static_cast(scoped)); + + SpanGuard const bare = std::move(scoped); + ASSERT_TRUE(static_cast(bare)); + + // Nothing exported yet: the span is still open. + EXPECT_EQ(countSpans(spanData()->GetSpans(), "ledger.build"), 0u); + + // 'bare' ends the span here on destruction; the moved-from 'scoped' + // guard is destroyed too but must NOT end it a second time. + } + + auto spans = spanData()->GetSpans(); + // Exactly one export: not zero (the guard still owns the span) and not two + // (the moved-from scoped guard does not re-end it). + EXPECT_EQ(countSpans(spans, "ledger.build"), 1u); +} + +// A ScopedSpanGuard's scope lives in the ACTIVE LocalValue store, and the +// coro-aware storage makes the ambient context follow that store. This +// simulates a coroutine that yields (its store swapped out for the worker's +// own) then resumes on another worker (store swapped back in): the span must be +// hidden while off-store, visible again on resume, and pop cleanly under the +// same store it was pushed on -- so the owner-store assertion never trips. +// +// Fails without CoroAwareContextStorage: OpenTelemetry's default thread-local +// stack ignores the LocalValue swap, so the span would stay visible off-store +// (the EXPECT_NE below would fail). Passes only because the fixture installs the +// coro-aware storage that binds the ambient stack to the active store. +TEST_F(SpanGuardScopeTest, scopedGuard_survives_localvalue_store_swap) +{ + namespace ctx = opentelemetry::context; + namespace trc = opentelemetry::trace; + + // Stand-in stores for the coroutine and a worker thread's own store. Both + // are onCoro=true so LocalValues::cleanup() never deletes these stack + // objects. Keeping a stack store active across every GetCurrent() call below + // stops LocalValue from materializing (then leaking) a heap thread-store. + xrpl::detail::LocalValues coroStore; + xrpl::detail::LocalValues workerStore; + + // Detach (do NOT delete) the fixture's active store, run on the coro store, + // and remember the original so teardown gets it back. + auto* saved = xrpl::detail::getLocalValues().release(); + xrpl::detail::getLocalValues().reset(&coroStore); + + trc::SpanContext captured = trc::SpanContext::GetInvalid(); + { + ScopedSpanGuard const span(TraceCategory::Rpc, "rpc", "process"); + ASSERT_TRUE(static_cast(span)); + auto active = trc::GetSpan(ctx::RuntimeContext::GetCurrent())->GetContext(); + ASSERT_TRUE(active.IsValid()); + captured = active; + + // Yield: swap the coro store OUT, the worker's own store IN. + xrpl::detail::getLocalValues().release(); + xrpl::detail::getLocalValues().reset(&workerStore); + auto offCoro = trc::GetSpan(ctx::RuntimeContext::GetCurrent())->GetContext(); + EXPECT_FALSE(offCoro.IsValid()); // no ambient span off-coro + EXPECT_NE(offCoro.span_id(), captured.span_id()); // span NOT visible + + // Resume on another worker: swap the coro store back IN. + xrpl::detail::getLocalValues().release(); + xrpl::detail::getLocalValues().reset(&coroStore); + auto resumed = trc::GetSpan(ctx::RuntimeContext::GetCurrent())->GetContext(); + EXPECT_TRUE(resumed.IsValid()); // ambient again + EXPECT_EQ(resumed.span_id(), captured.span_id()); // same span visible + } // ~ScopedSpanGuard pops from coroStore (its owner store active) -- no assert + + // The scope popped cleanly: the coro store's stack is empty again. + auto afterPop = trc::GetSpan(ctx::RuntimeContext::GetCurrent()); + EXPECT_FALSE(afterPop->GetContext().IsValid()); + + // Restore (re-own) the fixture's store for teardown before any stack store + // leaves scope, so the thread pointer never dangles. + xrpl::detail::getLocalValues().release(); + xrpl::detail::getLocalValues().reset(saved); + + // The span ended exactly once, when the scope popped on resume. + EXPECT_EQ(countSpans(spanData()->GetSpans(), "rpc.process"), 1u); +} + +// SpanGuard::activate() makes an already-owned span the ambient context for the +// activation's lifetime WITHOUT owning or ending it: before activate() the span +// is not ambient, during it the span is the current context, and after it the +// prior (empty) context is restored. ~ScopedActivation must NOT end the span -- +// the owning guard ends it exactly once. Identity is proved by matching the +// ambient span_id seen during activation to the single exported span. +TEST_F(SpanGuardScopeTest, activate_sets_ambient_without_owning) +{ + namespace ctx = opentelemetry::context; + namespace trc = opentelemetry::trace; + + auto guard = SpanGuard::freshRoot(TraceCategory::Transactions, "tx", "process"); + ASSERT_TRUE(static_cast(guard)); + + // Unscoped: the guard's span is NOT ambient before activate(). + EXPECT_FALSE(trc::GetSpan(ctx::RuntimeContext::GetCurrent())->GetContext().IsValid()); + + trc::SpanId activeId; + { + auto activation = guard.activate(); + auto active = trc::GetSpan(ctx::RuntimeContext::GetCurrent())->GetContext(); + EXPECT_TRUE(active.IsValid()); // now ambient + activeId = active.span_id(); + } // ~ScopedActivation pops the scope; it does NOT end the span. + + // A valid ambient span was observed while the activation was live. + EXPECT_TRUE(activeId.IsValid()); + + // Prior context restored: no ambient span after the activation drops. + EXPECT_FALSE(trc::GetSpan(ctx::RuntimeContext::GetCurrent())->GetContext().IsValid()); + + // Non-owning: the activation did NOT end the span, so nothing is exported. + EXPECT_EQ(countSpans(spanData()->GetSpans(), "tx.process"), 0u); + + // Ending the OWNING guard exports the span exactly once. + guard = SpanGuard{}; + auto spans = spanData()->GetSpans(); + EXPECT_EQ(countSpans(spans, "tx.process"), 1u); + + // The span made ambient during activation WAS the guard's own span. + auto* txSpan = findSpan(spans, "tx.process"); + ASSERT_NE(txSpan, nullptr); + EXPECT_EQ(txSpan->GetSpanId(), activeId); +} + +// A forced-root span started while a PendingTraceId is active adopts that +// pinned 16-byte trace_id and remains a true root (no parent). +TEST_F(SpanGuardScopeTest, deterministicIdGenerator_forced_root_gets_pending_trace_id) +{ + auto const h = makeTraceIdBytes(); + { + PendingTraceId const pending{h}; + auto rootCtx = opentelemetry::context::Context{opentelemetry::trace::kIsRootSpanKey, true}; + auto span = + telemetry_->startSpan("tx.receive", rootCtx, opentelemetry::trace::SpanKind::kInternal); + span->End(); + } // ~PendingTraceId asserts the id was consumed. + + auto spans = spanData()->GetSpans(); + ASSERT_EQ(spans.size(), 1u); + // trace_id == the pinned hash. + EXPECT_EQ(std::memcmp(spans[0]->GetTraceId().Id().data(), h.data(), 16), 0); + // TRUE ROOT: no parent. + EXPECT_FALSE(spans[0]->GetParentSpanId().IsValid()); +} + +// A forced-root span with NO PendingTraceId gets a random (non-zero) trace_id, +// never the deterministic hash -- the safety property when no id is pinned. +TEST_F(SpanGuardScopeTest, deterministicIdGenerator_no_pending_gives_random_root) +{ + auto const h = makeTraceIdBytes(); + { + auto rootCtx = opentelemetry::context::Context{opentelemetry::trace::kIsRootSpanKey, true}; + auto span = + telemetry_->startSpan("tx.receive", rootCtx, opentelemetry::trace::SpanKind::kInternal); + span->End(); + } + + auto spans = spanData()->GetSpans(); + ASSERT_EQ(spans.size(), 1u); + // Random root: valid (non-zero) trace_id... + EXPECT_TRUE(spans[0]->GetTraceId().IsValid()); + // ...and NOT the deterministic hash (no pending id leaked in). + EXPECT_NE(std::memcmp(spans[0]->GetTraceId().Id().data(), h.data(), 16), 0); + EXPECT_FALSE(spans[0]->GetParentSpanId().IsValid()); +} + +// SAFETY: an ambient child under a live parent never adopts a pending id. The +// SDK inherits the parent's trace_id and never calls GenerateTraceId() for the +// child, so the pinned id stays available -- proven here by a trailing +// forced-root span that DOES adopt it (which also consumes the id so +// ~PendingTraceId's consumed-assert holds; see the report for this choice). +TEST_F(SpanGuardScopeTest, deterministicIdGenerator_ambient_child_ignores_pending) +{ + auto const h = makeTraceIdBytes(); + { + // Ambient parent (a random-id root) active on this thread FIRST, before + // any id is pinned, so the parent itself does not consume it. + ScopedSpanGuard const parent(TraceCategory::Rpc, "rpc", "process"); + ASSERT_TRUE(static_cast(parent)); + + // Pin the id, then start an ambient child under the live parent. + PendingTraceId const pending{h}; + + // The child has a valid parent, so the SDK inherits the parent's + // trace_id and never calls GenerateTraceId(): the pending id is ignored. + { + auto child = parent.childSpan("rpc.dispatch"); + ASSERT_TRUE(static_cast(child)); + } + + // Consume the pinned id with a real forced-root span (its intended use), + // so ~PendingTraceId sees the id as consumed. Its trace_id == h. + { + auto rootCtx = + opentelemetry::context::Context{opentelemetry::trace::kIsRootSpanKey, true}; + auto root = telemetry_->startSpan( + "tx.receive", rootCtx, opentelemetry::trace::SpanKind::kInternal); + root->End(); + } + } // ~PendingTraceId: consumed == true, assert holds; then parent ends. + + auto spans = spanData()->GetSpans(); + auto* parent = findSpan(spans, "rpc.process"); + auto* child = findSpan(spans, "rpc.dispatch"); + auto* root = findSpan(spans, "tx.receive"); + ASSERT_NE(parent, nullptr); + ASSERT_NE(child, nullptr); + ASSERT_NE(root, nullptr); + + // The ambient child inherited the parent's trace and did NOT adopt h. + EXPECT_EQ(child->GetTraceId(), parent->GetTraceId()); + EXPECT_EQ(child->GetParentSpanId(), parent->GetSpanId()); + EXPECT_NE(std::memcmp(child->GetTraceId().Id().data(), h.data(), 16), 0); + + // The forced-root span DID adopt the pinned id: proof the id was available + // the whole time -- the ambient child simply never requested it. + EXPECT_EQ(std::memcmp(root->GetTraceId().Id().data(), h.data(), 16), 0); + EXPECT_FALSE(root->GetParentSpanId().IsValid()); +} + +// Death test guarding the cross-store scope-leak bug on ScopedSpanGuard. +// +// A ScopedSpanGuard's Scope is bound to the LocalValue context store that was +// active when it was constructed, so it must be destroyed while that same store +// is active. A different thread has a different LocalValue store, so destroying +// the guard on another thread pops the Scope against a foreign context stack -- +// ~ScopedSpanGuard's owner-store XRPL_ASSERT turns that silent corruption into a +// loud abort(). ScopedSpanGuard is non-movable, so it cannot be moved into a +// worker directly; instead we own it through a unique_ptr and move only that +// pointer to a worker thread, whose store differs from this thread's. +// +// The death happens on the WORKER thread (the moved-in unique_ptr is destroyed +// when the worker lambda's captures are torn down, before worker.join() +// completes). A failed assert() calls abort(), which raises SIGABRT +// process-wide regardless of thread, so EXPECT_DEATH -- which runs the +// statement in a forked child and checks it dies -- observes the crash. The +// regex matches the assert message substring; the assert message is written as +// two adjacent string literals in SpanGuard.cpp, so the ".*" bridges the gap +// between "the" and "constructing". +// +// The test is skipped where the assertion cannot fire: under NDEBUG (Release +// builds) XRPL_ASSERT is a no-op, and under ENABLE_VOIDSTAR a failed assert +// continues instead of aborting -- in both cases the worker would not crash and +// EXPECT_DEATH would report a spurious failure. +TEST_F(SpanGuardScopeTest, scopedGuard_cross_thread_death_asserts_at_wrong_store_destroy) +{ +#ifdef NDEBUG + GTEST_SKIP() << "XRPL_ASSERT compiles to a no-op under NDEBUG (Release builds), so the " + "cross-store scope-leak assertion this test exercises does not fire."; +#elifdef ENABLE_VOIDSTAR + GTEST_SKIP() << "ENABLE_VOIDSTAR continues past a failed XRPL_ASSERT instead of aborting, so " + "the cross-store scope-leak assertion this test exercises does not crash."; +#else + EXPECT_DEATH( + { + // Scoped guard constructed on THIS thread; its Scope binds to this + // thread's LocalValue store. + auto scoped = + std::make_unique(TraceCategory::Ledger, "ledger", "build"); + + // Move only the owning pointer to a worker. ~ScopedSpanGuard runs on + // the worker when the lambda's captures are destroyed; the worker's + // store differs from the constructing store, tripping the owner-store + // assertion -> abort(). + std::thread worker([s = std::move(scoped)]() mutable {}); + worker.join(); + }, + // The assert message is written as two adjacent string literals in + // SpanGuard.cpp; assert() stringifies the expression source via the + // preprocessor '#' operator, keeping both quoted literals with the + // "\" \"" gap between "the" and "constructing". Match across that gap. + ".*destroyed on.*constructing context store.*"); +#endif +} + +} // namespace +} // namespace xrpl::telemetry + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/TelemetryConfig.cpp b/src/tests/libxrpl/telemetry/TelemetryConfig.cpp new file mode 100644 index 00000000000..bfa0b851495 --- /dev/null +++ b/src/tests/libxrpl/telemetry/TelemetryConfig.cpp @@ -0,0 +1,146 @@ +#include +#include +#include + +#include + +#include + +using namespace xrpl; + +TEST(TelemetryConfig, setup_defaults) +{ + telemetry::Telemetry::Setup const s; + EXPECT_FALSE(s.enabled); + EXPECT_EQ(s.serviceName, "xrpld"); + EXPECT_TRUE(s.serviceVersion.empty()); + EXPECT_TRUE(s.serviceInstanceId.empty()); + EXPECT_EQ(s.exporterEndpoint, "http://localhost:4318/v1/traces"); + EXPECT_FALSE(s.useTls); + EXPECT_TRUE(s.tlsCertPath.empty()); + EXPECT_DOUBLE_EQ(s.samplingRatio, 1.0); + EXPECT_EQ(s.batchSize, 512u); + EXPECT_EQ(s.batchDelay, std::chrono::milliseconds{5000}); + EXPECT_EQ(s.maxQueueSize, 2048u); + EXPECT_EQ(s.networkId, 0u); + EXPECT_EQ(s.networkType, "mainnet"); + EXPECT_TRUE(s.traceTransactions); + EXPECT_TRUE(s.traceConsensus); + EXPECT_TRUE(s.traceRpc); + EXPECT_TRUE(s.tracePeer); + EXPECT_TRUE(s.traceLedger); +} + +TEST(TelemetryConfig, parse_empty_section) +{ + Section const section; + auto setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0); + + EXPECT_FALSE(setup.enabled); + EXPECT_EQ(setup.serviceName, "xrpld"); + EXPECT_EQ(setup.serviceVersion, "2.0.0"); + EXPECT_EQ(setup.serviceInstanceId, "nHUtest123"); + EXPECT_DOUBLE_EQ(setup.samplingRatio, 1.0); + EXPECT_TRUE(setup.traceRpc); + EXPECT_TRUE(setup.traceTransactions); + EXPECT_TRUE(setup.traceConsensus); + EXPECT_TRUE(setup.tracePeer); + EXPECT_TRUE(setup.traceLedger); +} + +TEST(TelemetryConfig, parse_full_section) +{ + Section section; + section.set("enabled", "1"); + section.set("service_name", "my-rippled"); + section.set("service_instance_id", "custom-id"); + section.set("exporter", "otlp_http"); + section.set("endpoint", "http://collector:4318/v1/traces"); + section.set("use_tls", "1"); + section.set("tls_ca_cert", "/etc/ssl/ca.pem"); + section.set("batch_size", "256"); + section.set("batch_delay_ms", "3000"); + section.set("max_queue_size", "4096"); + section.set("trace_transactions", "0"); + section.set("trace_consensus", "0"); + section.set("trace_rpc", "1"); + section.set("trace_peer", "1"); + section.set("trace_ledger", "0"); + + auto setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 1); + + EXPECT_TRUE(setup.enabled); + EXPECT_EQ(setup.serviceName, "my-rippled"); + EXPECT_EQ(setup.serviceInstanceId, "custom-id"); + EXPECT_EQ(setup.exporterEndpoint, "http://collector:4318/v1/traces"); + EXPECT_TRUE(setup.useTls); + EXPECT_EQ(setup.tlsCertPath, "/etc/ssl/ca.pem"); + EXPECT_EQ(setup.batchSize, 256u); + EXPECT_EQ(setup.batchDelay, std::chrono::milliseconds{3000}); + EXPECT_EQ(setup.maxQueueSize, 4096u); + EXPECT_FALSE(setup.traceTransactions); + EXPECT_FALSE(setup.traceConsensus); + EXPECT_TRUE(setup.traceRpc); + EXPECT_TRUE(setup.tracePeer); + EXPECT_FALSE(setup.traceLedger); +} + +TEST(TelemetryConfig, mtls_cert_and_key_both_set) +{ + Section section; + section.set("use_tls", "1"); + section.set("tls_client_cert", "/etc/ssl/client.pem"); + section.set("tls_client_key", "/etc/ssl/client.key"); + + auto setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0); + EXPECT_EQ(setup.tlsClientCertPath, "/etc/ssl/client.pem"); + EXPECT_EQ(setup.tlsClientKeyPath, "/etc/ssl/client.key"); +} + +TEST(TelemetryConfig, mtls_cert_without_key_throws) +{ + Section section; + section.set("tls_client_cert", "/etc/ssl/client.pem"); + EXPECT_THROW( + telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0), std::runtime_error); +} + +TEST(TelemetryConfig, mtls_key_without_cert_throws) +{ + Section section; + section.set("tls_client_key", "/etc/ssl/client.key"); + EXPECT_THROW( + telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0), std::runtime_error); +} + +TEST(TelemetryConfig, mtls_neither_set_is_one_way_tls) +{ + Section section; + section.set("use_tls", "1"); + section.set("tls_ca_cert", "/etc/ssl/ca.pem"); + + auto setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0); + EXPECT_TRUE(setup.tlsClientCertPath.empty()); + EXPECT_TRUE(setup.tlsClientKeyPath.empty()); +} + +TEST(TelemetryConfig, null_telemetry_factory) +{ + telemetry::Telemetry::Setup setup; + setup.enabled = false; + + beast::Journal::Sink& sink = beast::Journal::getNullSink(); + beast::Journal const j(sink); + auto tel = telemetry::makeTelemetry(setup, j); + EXPECT_TRUE(tel != nullptr); + EXPECT_FALSE(tel->isEnabled()); + EXPECT_FALSE(tel->shouldTraceRpc()); + EXPECT_FALSE(tel->shouldTraceTransactions()); + EXPECT_FALSE(tel->shouldTraceConsensus()); + EXPECT_FALSE(tel->shouldTracePeer()); + EXPECT_FALSE(tel->shouldTraceLedger()); + + // start/stop should be no-ops without crashing + tel->start(); + tel->stop(); +} diff --git a/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp b/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp new file mode 100644 index 00000000000..fdc6c443cb3 --- /dev/null +++ b/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp @@ -0,0 +1,158 @@ +#include + +#ifdef XRPL_ENABLE_TELEMETRY + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace trace = opentelemetry::trace; + +TEST(TraceContextPropagator, round_trip) +{ + std::uint8_t traceIdBuf[16] = { + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0a, + 0x0b, + 0x0c, + 0x0d, + 0x0e, + 0x0f, + 0x10}; + std::uint8_t spanIdBuf[8] = {0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x11, 0x22}; + + trace::TraceId const traceId( + opentelemetry::nostd::span(traceIdBuf, 16)); + trace::SpanId const spanId(opentelemetry::nostd::span(spanIdBuf, 8)); + trace::TraceFlags const flags(trace::TraceFlags::kIsSampled); + trace::SpanContext const spanCtx(traceId, spanId, flags, true); + + auto ctx = opentelemetry::context::Context{}.SetValue( + trace::kSpanKey, + opentelemetry::nostd::shared_ptr(new trace::DefaultSpan(spanCtx))); + + protocol::TraceContext proto; + xrpl::telemetry::injectToProtobuf(ctx, proto); + + EXPECT_TRUE(proto.has_trace_id()); + EXPECT_EQ(proto.trace_id().size(), 16u); + EXPECT_TRUE(proto.has_span_id()); + EXPECT_EQ(proto.span_id().size(), 8u); + EXPECT_EQ(proto.trace_flags(), static_cast(trace::TraceFlags::kIsSampled)); + EXPECT_EQ(std::memcmp(proto.trace_id().data(), traceIdBuf, 16), 0); + EXPECT_EQ(std::memcmp(proto.span_id().data(), spanIdBuf, 8), 0); + + auto extractedCtx = xrpl::telemetry::extractFromProtobuf(proto); + auto extractedSpan = trace::GetSpan(extractedCtx); + ASSERT_NE(extractedSpan, nullptr); + + auto const& extracted = extractedSpan->GetContext(); + EXPECT_TRUE(extracted.IsValid()); + EXPECT_TRUE(extracted.IsRemote()); + EXPECT_EQ(extracted.trace_id(), traceId); + EXPECT_EQ(extracted.span_id(), spanId); + EXPECT_TRUE(extracted.trace_flags().IsSampled()); +} + +TEST(TraceContextPropagator, extract_empty_protobuf) +{ + protocol::TraceContext const proto; + auto ctx = xrpl::telemetry::extractFromProtobuf(proto); + auto span = trace::GetSpan(ctx); + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().IsValid()); +} + +TEST(TraceContextPropagator, extract_wrong_size_trace_id) +{ + protocol::TraceContext proto; + proto.set_trace_id(std::string(8, '\x01')); + proto.set_span_id(std::string(8, '\xaa')); + + auto ctx = xrpl::telemetry::extractFromProtobuf(proto); + auto span = trace::GetSpan(ctx); + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().IsValid()); +} + +TEST(TraceContextPropagator, extract_wrong_size_span_id) +{ + protocol::TraceContext proto; + proto.set_trace_id(std::string(16, '\x01')); + proto.set_span_id(std::string(4, '\xaa')); + + auto ctx = xrpl::telemetry::extractFromProtobuf(proto); + auto span = trace::GetSpan(ctx); + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().IsValid()); +} + +TEST(TraceContextPropagator, inject_invalid_span) +{ + auto ctx = opentelemetry::context::Context{}; + protocol::TraceContext proto; + xrpl::telemetry::injectToProtobuf(ctx, proto); + + EXPECT_FALSE(proto.has_trace_id()); + EXPECT_FALSE(proto.has_span_id()); +} + +TEST(TraceContextPropagator, flags_preservation) +{ + std::uint8_t const traceIdBuf[16] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; + std::uint8_t const spanIdBuf[8] = {1, 2, 3, 4, 5, 6, 7, 8}; + + // Test with flags NOT sampled (flags = 0) + trace::TraceFlags const flags(0); + trace::SpanContext const spanCtx( + trace::TraceId(opentelemetry::nostd::span(traceIdBuf, 16)), + trace::SpanId(opentelemetry::nostd::span(spanIdBuf, 8)), + flags, + true); + + auto ctx = opentelemetry::context::Context{}.SetValue( + trace::kSpanKey, + opentelemetry::nostd::shared_ptr(new trace::DefaultSpan(spanCtx))); + + protocol::TraceContext proto; + xrpl::telemetry::injectToProtobuf(ctx, proto); + EXPECT_EQ(proto.trace_flags(), 0u); + + auto extracted = xrpl::telemetry::extractFromProtobuf(proto); + auto span = trace::GetSpan(extracted); + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().trace_flags().IsSampled()); +} + +#else // XRPL_ENABLE_TELEMETRY not defined + +TEST(TraceContextPropagator, compiles_without_telemetry) +{ + SUCCEED(); +} + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/TxApplySpanNames.cpp b/src/tests/libxrpl/telemetry/TxApplySpanNames.cpp new file mode 100644 index 00000000000..ad6bb4efa8b --- /dev/null +++ b/src/tests/libxrpl/telemetry/TxApplySpanNames.cpp @@ -0,0 +1,62 @@ +#include + +#include + +#include + +/** + * Contract tests for the transaction apply-pipeline span constants. + * + * The span names and attribute keys in TxApplySpanNames.h are a cross-component + * contract: the collector spanmetrics connector aggregates on these exact + * strings (dimensions tx_type, ter_result, stage) and the Grafana + * transaction-overview dashboard queries them. A silent rename here would + * break per-stage metrics with no compile error, so these tests pin the + * literal values. They need no telemetry runtime and run in every build. + */ + +using namespace xrpl::telemetry; + +TEST(TxApplySpanNames, span_names_are_dot_qualified) +{ + // Full span names feed SpanGuard::hashSpan() in applySteps.cpp. + EXPECT_EQ(std::string_view(tx_apply_span::preflight), "tx.preflight"); + EXPECT_EQ(std::string_view(tx_apply_span::preclaim), "tx.preclaim"); +} + +TEST(TxApplySpanNames, operation_suffixes) +{ + // Suffix used with SpanGuard::span(cat, seg::tx, suffix) in Transactor.cpp. + EXPECT_EQ(std::string_view(tx_apply_span::op::preflight), "preflight"); + EXPECT_EQ(std::string_view(tx_apply_span::op::preclaim), "preclaim"); + EXPECT_EQ(std::string_view(tx_apply_span::op::transactor), "transactor"); +} + +TEST(TxApplySpanNames, attribute_keys_match_collector_dimensions) +{ + // These keys MUST match docker/telemetry/otel-collector-config.yaml + // spanmetrics dimensions and TxSpanNames.h (so both span sets aggregate + // under one dimension). + EXPECT_EQ(std::string_view(tx_apply_span::attr::stage), "stage"); + EXPECT_EQ(std::string_view(tx_apply_span::attr::txType), "tx_type"); + EXPECT_EQ(std::string_view(tx_apply_span::attr::terResult), "ter_result"); + EXPECT_EQ(std::string_view(tx_apply_span::attr::applied), "applied"); +} + +TEST(TxApplySpanNames, stage_values_are_the_three_pipeline_stages) +{ + // The stage attribute carries exactly these three values; they become the + // spanmetrics `stage` dimension cardinality (3) and the dashboard filter. + EXPECT_EQ(std::string_view(tx_apply_span::val::preflight), "preflight"); + EXPECT_EQ(std::string_view(tx_apply_span::val::preclaim), "preclaim"); + EXPECT_EQ(std::string_view(tx_apply_span::val::apply), "apply"); +} + +TEST(TxApplySpanNames, current_ledger_keys_are_shared_constants) +{ + // "ledger being worked on" keys, re-exported from the base SpanNames.h so + // every emitter (tx.*, txq.*, apply pipeline) shares one string. Set on + // preclaim/transactor (view-bearing); preflight omits them (stateless). + EXPECT_EQ(std::string_view(tx_apply_span::attr::currentLedgerSeq), "current_ledger_seq"); + EXPECT_EQ(std::string_view(tx_apply_span::attr::currentLedgerHash), "current_ledger_hash"); +} diff --git a/src/tests/libxrpl/telemetry/ValidationTracker.cpp b/src/tests/libxrpl/telemetry/ValidationTracker.cpp new file mode 100644 index 00000000000..2542316e7a5 --- /dev/null +++ b/src/tests/libxrpl/telemetry/ValidationTracker.cpp @@ -0,0 +1,379 @@ +/** + * @file ValidationTracker.cpp + * Unit tests for xrpl::telemetry::ValidationTracker. + */ + +#include + +#include +#include + +#include + +#include +#include +#include +#include + +using namespace xrpl; +using namespace xrpl::telemetry; + +/** + * Helper to create a unique uint256 from an integer seed. + */ +static uint256 +makeHash(std::uint64_t n) +{ + return uint256(n); +} + +/** + * Test fixture providing a fresh ValidationTracker per test. + */ +class ValidationTrackerTest : public ::testing::Test +{ +protected: + ValidationTracker tracker_; +}; + +// --------------------------------------------------------------- +// 1. Normal agreement +// Record both our validation and network validation for the +// same hash, then reconcile after the grace period elapses. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, NormalAgreement) +{ + auto const hash = makeHash(1); + LedgerIndex const seq = 100; + + tracker_.recordOurValidation(hash, seq); + tracker_.recordNetworkValidation(hash, seq); + + // Immediately after recording, nothing is reconciled yet + // (grace period has not elapsed). + tracker_.reconcile(); + EXPECT_EQ(tracker_.totalValidationsSent(), 1u); + EXPECT_EQ(tracker_.totalValidationsChecked(), 1u); + + // Wait for the grace period (8 seconds) to elapse, then reconcile. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + EXPECT_EQ(tracker_.totalAgreements(), 1u); + EXPECT_EQ(tracker_.totalMissed(), 0u); + EXPECT_EQ(tracker_.agreements1h(), 1u); + EXPECT_EQ(tracker_.missed1h(), 0u); + EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 100.0); +} + +// --------------------------------------------------------------- +// 2. Missed validation +// Only the network validates; we never do. After grace period +// the event should be reconciled as a miss. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, MissedValidation) +{ + auto const hash = makeHash(2); + LedgerIndex const seq = 200; + + tracker_.recordNetworkValidation(hash, seq); + + // Wait for grace period then reconcile. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + EXPECT_EQ(tracker_.totalAgreements(), 0u); + EXPECT_EQ(tracker_.totalMissed(), 1u); + EXPECT_EQ(tracker_.agreements1h(), 0u); + EXPECT_EQ(tracker_.missed1h(), 1u); + EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 0.0); +} + +// --------------------------------------------------------------- +// 3. Late repair +// Network validates first, grace period elapses (miss), then +// our validation arrives within the 5-minute repair window and +// the miss is flipped to an agreement. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, LateRepair) +{ + auto const hash = makeHash(3); + LedgerIndex const seq = 300; + + // Network validates, but we do not (yet). + tracker_.recordNetworkValidation(hash, seq); + + // Grace period elapses -- reconciled as a miss. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + EXPECT_EQ(tracker_.totalMissed(), 1u); + EXPECT_EQ(tracker_.totalAgreements(), 0u); + EXPECT_EQ(tracker_.missed1h(), 1u); + + // Late arrival of our validation (within repair window). + tracker_.recordOurValidation(hash, seq); + tracker_.reconcile(); + + // Miss should be repaired to agreement. + EXPECT_EQ(tracker_.totalAgreements(), 1u); + EXPECT_EQ(tracker_.totalMissed(), 0u); + EXPECT_EQ(tracker_.agreements1h(), 1u); + EXPECT_EQ(tracker_.missed1h(), 0u); + EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 100.0); +} + +// --------------------------------------------------------------- +// 4. Empty window returns 0% +// When no events have been recorded the percentage methods +// must return 0.0, not NaN or any other value. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, EmptyWindowReturnsZero) +{ + EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 0.0); + EXPECT_DOUBLE_EQ(tracker_.agreementPct24h(), 0.0); + EXPECT_EQ(tracker_.agreements1h(), 0u); + EXPECT_EQ(tracker_.missed1h(), 0u); + EXPECT_EQ(tracker_.agreements24h(), 0u); + EXPECT_EQ(tracker_.missed24h(), 0u); + EXPECT_EQ(tracker_.totalAgreements(), 0u); + EXPECT_EQ(tracker_.totalMissed(), 0u); + EXPECT_EQ(tracker_.totalAgreementsEver(), 0u); + EXPECT_EQ(tracker_.totalMissedEver(), 0u); + EXPECT_EQ(tracker_.totalValidationsSent(), 0u); + EXPECT_EQ(tracker_.totalValidationsChecked(), 0u); +} + +// --------------------------------------------------------------- +// 5. Grace period boundary +// Events recorded less than 8 seconds ago must NOT be +// reconciled. Verify that an immediate reconcile is a no-op. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, GracePeriodBoundary) +{ + auto const hash = makeHash(5); + LedgerIndex const seq = 500; + + tracker_.recordOurValidation(hash, seq); + tracker_.recordNetworkValidation(hash, seq); + + // Reconcile immediately -- grace period has not elapsed. + tracker_.reconcile(); + + // Nothing should be reconciled yet. + EXPECT_EQ(tracker_.totalAgreements(), 0u); + EXPECT_EQ(tracker_.totalMissed(), 0u); + EXPECT_EQ(tracker_.agreements1h(), 0u); + EXPECT_EQ(tracker_.missed1h(), 0u); + + // Lifetime send/check counters should still be incremented. + EXPECT_EQ(tracker_.totalValidationsSent(), 1u); + EXPECT_EQ(tracker_.totalValidationsChecked(), 1u); +} + +// --------------------------------------------------------------- +// 6. Max pending events -- trimming +// Add more than kMaxPendingEvents (1000) events. After +// reconciliation and a second reconcile pass the pending map +// should be trimmed. Lifetime totals must remain consistent. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, MaxPendingEventsTrimming) +{ + constexpr std::size_t kCount = 1100; + + for (std::size_t i = 0; i < kCount; ++i) + { + auto const hash = makeHash(i + 1); + auto const seq = static_cast(i + 1); + tracker_.recordOurValidation(hash, seq); + tracker_.recordNetworkValidation(hash, seq); + } + + EXPECT_EQ(tracker_.totalValidationsSent(), kCount); + EXPECT_EQ(tracker_.totalValidationsChecked(), kCount); + + // Wait for grace period so all events can be reconciled. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + // All events should be reconciled as agreements. + EXPECT_EQ(tracker_.totalAgreements(), kCount); + EXPECT_EQ(tracker_.totalMissed(), 0u); + + // Reconcile again to trigger pending eviction / trimming. + // The pending map should be trimmed, but totals remain correct. + tracker_.reconcile(); + EXPECT_EQ(tracker_.totalAgreements(), kCount); + EXPECT_EQ(tracker_.totalMissed(), 0u); +} + +// --------------------------------------------------------------- +// 7. Multiple distinct ledgers -- mixed results +// Record a mix of agreements and misses to verify that window +// counts and percentages are computed correctly. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, MixedAgreementsAndMisses) +{ + // 3 agreements: both sides validate. + for (int i = 1; i <= 3; ++i) + { + auto const hash = makeHash(static_cast(i)); + tracker_.recordOurValidation(hash, static_cast(i)); + tracker_.recordNetworkValidation(hash, static_cast(i)); + } + + // 2 misses: only network validates. + for (int i = 4; i <= 5; ++i) + { + auto const hash = makeHash(static_cast(i)); + tracker_.recordNetworkValidation(hash, static_cast(i)); + } + + // Wait for grace period then reconcile. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + EXPECT_EQ(tracker_.totalAgreements(), 3u); + EXPECT_EQ(tracker_.totalMissed(), 2u); + EXPECT_EQ(tracker_.agreements1h(), 3u); + EXPECT_EQ(tracker_.missed1h(), 2u); + + // 3 out of 5 = 60% + EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 60.0); +} + +// --------------------------------------------------------------- +// 8. Duplicate recording for same hash +// Recording the same hash multiple times should not create +// duplicate pending entries or double-count totals beyond the +// per-call increments. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, DuplicateRecordingSameHash) +{ + auto const hash = makeHash(42); + LedgerIndex const seq = 42; + + // Record our validation twice for the same hash. + tracker_.recordOurValidation(hash, seq); + tracker_.recordOurValidation(hash, seq); + tracker_.recordNetworkValidation(hash, seq); + + // Each call increments the lifetime counter. + EXPECT_EQ(tracker_.totalValidationsSent(), 2u); + EXPECT_EQ(tracker_.totalValidationsChecked(), 1u); + + // But only one pending event exists, so only one agreement. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + EXPECT_EQ(tracker_.totalAgreements(), 1u); + EXPECT_EQ(tracker_.totalMissed(), 0u); +} + +// --------------------------------------------------------------- +// 9. Only-we-validated scenario +// We validate but the network does not. After grace period +// this should be a miss (not an agreement). +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, OnlyWeValidated) +{ + auto const hash = makeHash(99); + LedgerIndex const seq = 99; + + tracker_.recordOurValidation(hash, seq); + + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + EXPECT_EQ(tracker_.totalAgreements(), 0u); + EXPECT_EQ(tracker_.totalMissed(), 1u); + EXPECT_EQ(tracker_.missed1h(), 1u); + EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 0.0); +} + +// --------------------------------------------------------------- +// 10. Gross miss tally is monotonic across a late repair +// The gross lifetime tallies (totalAgreementsEver/totalMissedEver) +// back the monotonic Prometheus _total counters. A late repair must +// move the NET totals (miss -> agreement) but must NOT move the gross +// tallies: a miss already counted stays counted, and the repair does +// not add a second (agreement) count for the same ledger. +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, GrossMissedNeverDecrementsOnRepair) +{ + auto const hash = makeHash(10); + LedgerIndex const seq = 1000; + + // Network validates, we do not (yet). + tracker_.recordNetworkValidation(hash, seq); + + // Grace period elapses -- reconciled as a miss. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + // Net and gross both show exactly one initial miss, zero agreements. + EXPECT_EQ(tracker_.totalMissed(), 1u); + EXPECT_EQ(tracker_.totalMissedEver(), 1u); + EXPECT_EQ(tracker_.totalAgreements(), 0u); + EXPECT_EQ(tracker_.totalAgreementsEver(), 0u); + + // Late arrival of our validation repairs the miss to an agreement. + tracker_.recordOurValidation(hash, seq); + tracker_.reconcile(); + + // Net totals reflect the repair... + EXPECT_EQ(tracker_.totalMissed(), 0u); + EXPECT_EQ(tracker_.totalAgreements(), 1u); + // ...but the gross tallies are frozen at first classification: the miss + // stays counted and no agreement was added (repair path excluded). + EXPECT_EQ(tracker_.totalMissedEver(), 1u); + EXPECT_EQ(tracker_.totalAgreementsEver(), 0u); +} + +// --------------------------------------------------------------- +// 11. Gross tallies count initial classification only (additive) +// With a mix of initial agreements and misses the gross tallies equal +// the net totals. A subsequent repair shifts the net totals but leaves +// the gross tallies unchanged, and the gross sum equals the number of +// reconciled ledgers (the additive invariant the _total counters rely on). +// --------------------------------------------------------------- +TEST_F(ValidationTrackerTest, GrossAgreementsCountInitialOnly) +{ + // 3 initial agreements: both sides validate. + for (int i = 1; i <= 3; ++i) + { + auto const h = makeHash(static_cast(i)); + tracker_.recordOurValidation(h, static_cast(i)); + tracker_.recordNetworkValidation(h, static_cast(i)); + } + + // 2 initial misses: only network validates. + for (int i = 4; i <= 5; ++i) + { + auto const h = makeHash(static_cast(i)); + tracker_.recordNetworkValidation(h, static_cast(i)); + } + + // Grace period elapses -- all five reconciled at first classification. + std::this_thread::sleep_for(std::chrono::seconds(9)); + tracker_.reconcile(); + + // Before any repair, gross equals net. + EXPECT_EQ(tracker_.totalAgreements(), 3u); + EXPECT_EQ(tracker_.totalAgreementsEver(), 3u); + EXPECT_EQ(tracker_.totalMissed(), 2u); + EXPECT_EQ(tracker_.totalMissedEver(), 2u); + + // Repair one of the misses (hash 4) within the repair window. + tracker_.recordOurValidation(makeHash(4), 4); + tracker_.reconcile(); + + // Net totals shift by the repair... + EXPECT_EQ(tracker_.totalAgreements(), 4u); + EXPECT_EQ(tracker_.totalMissed(), 1u); + // ...gross tallies stay at the initial classification. + EXPECT_EQ(tracker_.totalAgreementsEver(), 3u); + EXPECT_EQ(tracker_.totalMissedEver(), 2u); + + // Additive invariant: gross agree + gross miss == ledgers reconciled. + EXPECT_EQ(tracker_.totalAgreementsEver() + tracker_.totalMissedEver(), 5u); +} diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 4abf77f5786..250b2ac5f37 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -18,9 +18,12 @@ #include #include #include +#include #include #include #include +#include +#include #include #include @@ -62,6 +65,9 @@ #include #include #include +#include +#include +#include #include @@ -80,6 +86,7 @@ #include #include #include +#include #include #include #include @@ -225,6 +232,14 @@ RCLConsensus::Adaptor::share(RCLCxTx const& tx) void RCLConsensus::Adaptor::propose(RCLCxPeerPos::Proposal const& proposal) { + // Child of the round span via its captured context (roundSpan_ is a + // thread-free SpanGuard, so parent explicitly via its context). + auto span = telemetry::SpanGuard::childSpan( + telemetry::consensus::span::proposalSend, roundSpanContext_); + span.setAttribute( + telemetry::consensus::span::attr::round, static_cast(proposal.proposeSeq())); + span.setAttribute(telemetry::consensus::span::attr::isBowOut, proposal.isBowOut()); + JLOG(j_.trace()) << (proposal.isBowOut() ? "We bow out: " : "We propose: ") << xrpl::to_string(proposal.prevLedger()) << " -> " << xrpl::to_string(proposal.position()); @@ -261,6 +276,11 @@ RCLConsensus::Adaptor::propose(RCLCxPeerPos::Proposal const& proposal) app_.getHashRouter().addSuppression(suppression); + // Inject the current thread's active span context (e.g. the + // consensus round span from Phase 4) so receiving peers can link + // their proposal.receive span as a child of this trace. + telemetry::SpanGuard::injectCurrentContextToProtobuf(*prop.mutable_trace_context()); + app_.getOverlay().broadcast(prop); } @@ -327,6 +347,21 @@ RCLConsensus::Adaptor::onClose( NetClock::time_point const& closeTime, ConsensusMode mode) -> Result { + namespace cs = telemetry::consensus::span; + + // Child of the round span via its captured context (roundSpan_ is a + // thread-free SpanGuard, so parent explicitly via its context). + auto span = telemetry::SpanGuard::childSpan(cs::ledgerClose, roundSpanContext_); + span.setAttribute(cs::attr::ledgerSeq, static_cast(ledger.ledger->header().seq) + 1); + span.setAttribute(cs::attr::mode, toDisplayString(mode).c_str()); + span.setAttribute( + cs::attr::txCountOpen, static_cast(app_.getOpenLedger().current()->txCount())); + span.setAttribute( + cs::attr::closeTimeResolutionMs, + static_cast( + std::chrono::duration_cast(ledger.closeTimeResolution()) + .count())); + bool const wrongLCL = mode == ConsensusMode::WrongLedger; bool const proposing = mode == ConsensusMode::Proposing; @@ -422,7 +457,15 @@ RCLConsensus::Adaptor::onForceAccept( ConsensusMode const& mode, json::Value&& consensusJson) { - doAccept(result, prevLedger, closeResolution, rawCloseTimes, mode, std::move(consensusJson)); + auto acceptSpan = makeAcceptSpan(result); + doAccept( + result, + prevLedger, + closeResolution, + rawCloseTimes, + mode, + std::move(consensusJson), + std::move(acceptSpan)); } void @@ -435,22 +478,74 @@ RCLConsensus::Adaptor::onAccept( json::Value&& consensusJson, bool const validating) { + auto acceptSpan = makeAcceptSpan(result); + app_.getJobQueue().addJob( JtAccept, "AcceptLedger", // NOLINTNEXTLINE(cppcoreguidelines-misleading-capture-default-by-value) - [=, this, cj = std::move(consensusJson)]() mutable { + [=, this, cj = std::move(consensusJson), sp = std::move(acceptSpan)]() mutable { // Note that no lock is held or acquired during this job. // This is because generic Consensus guarantees that once a ledger // is accepted, the consensus results and capture by reference state // will not change until startRound is called (which happens via // endConsensus). RclConsensusLogger clog("onAccept", validating, j_); - this->doAccept(result, prevLedger, closeResolution, rawCloseTimes, mode, std::move(cj)); + this->doAccept( + result, + prevLedger, + closeResolution, + rawCloseTimes, + mode, + std::move(cj), + std::move(sp)); this->app_.getOPs().endConsensus(clog.ss()); }); } +std::shared_ptr +RCLConsensus::Adaptor::makeAcceptSpan(Result const& result) +{ + namespace cs = telemetry::consensus::span; + + auto span = std::make_shared( + telemetry::SpanGuard::childSpan(cs::accept, roundSpanContext_)); + span->setAttribute(cs::attr::proposers, static_cast(result.proposers)); + span->setAttribute( + cs::attr::roundTimeMs, static_cast(result.roundTime.read().count())); + span->setAttribute(cs::attr::quorum, static_cast(app_.getValidators().quorum())); + span->setAttribute(cs::attr::disputesCount, static_cast(result.disputes.size())); + char const* stateStr = [&] { + switch (result.state) + { + case ConsensusState::Yes: + return "yes"; + case ConsensusState::MovedOn: + return "moved_on"; + case ConsensusState::Expired: + return "expired"; + default: + return "no"; + } + }(); + span->setAttribute(cs::attr::consensusState, stateStr); + + // Capture the accept span's context so createValidationSpan() — which + // runs on the jtACCEPT worker thread — can link the validation.send + // span to the accept span (matching the design diagram and the + // "validation follows acceptance" causal model). + if (*span) + { + // span is a thread-free SpanGuard handed to the JtAccept worker + // (onAccept), which ends it there. spanContext() captures the guard's + // own span, so accept.apply parents via acceptSpanContext_ regardless + // of which thread runs the sync (onForceAccept) or async (onAccept) + // path -- no scope work is needed. + acceptSpanContext_ = span->spanContext(); + } + return span; +} + void RCLConsensus::Adaptor::doAccept( Result const& result, @@ -458,8 +553,17 @@ RCLConsensus::Adaptor::doAccept( NetClock::duration closeResolution, ConsensusCloseTimes const& rawCloseTimes, ConsensusMode const& mode, - json::Value&& consensusJson) + json::Value&& consensusJson, + std::shared_ptr acceptSpan) { + namespace cs = telemetry::consensus::span; + + // Make the accept span ambient for the whole accept so doAccept's log lines + // (and any spans created here) correlate to it. Non-owning: acceptSpan still + // owns/ends the span. doAccept runs to completion on the JtAccept worker + // (no coroutine yield), so this scope is thread-local and safe. + auto acceptActivation = telemetry::activateIfLive(acceptSpan); + prevProposers_ = result.proposers; prevRoundTime_ = result.roundTime.read(); @@ -486,6 +590,53 @@ RCLConsensus::Adaptor::doAccept( closeTimeCorrect = true; } + // Parent accept.apply via the captured accept context (acceptSpanContext_): + // the accept span is a thread-free SpanGuard, so an explicit context is + // used for both the sync (onForceAccept) and async (onAccept) paths. Falls + // back to the round context if the accept span was null. + auto doAcceptSpan = acceptSpanContext_.isValid() + ? telemetry::SpanGuard::childSpan(cs::acceptApply, acceptSpanContext_) + : telemetry::SpanGuard::childSpan(cs::acceptApply, roundSpanContext_); + doAcceptSpan.setAttribute(cs::attr::ledgerSeq, static_cast(prevLedger.seq()) + 1); + doAcceptSpan.setAttribute( + cs::attr::closeTime, static_cast(consensusCloseTime.time_since_epoch().count())); + doAcceptSpan.setAttribute(cs::attr::closeTimeCorrect, closeTimeCorrect); + doAcceptSpan.setAttribute( + cs::attr::closeResolutionMs, + static_cast( + std::chrono::duration_cast(closeResolution).count())); + doAcceptSpan.setAttribute( + cs::attr::consensusState, + consensusFail ? std::string_view{cs::val::movedOn} : std::string_view{cs::val::finished}); + doAcceptSpan.setAttribute(cs::attr::proposing, proposing); + doAcceptSpan.setAttribute( + cs::attr::roundTimeMs, static_cast(result.roundTime.read().count())); + doAcceptSpan.setAttribute( + cs::attr::parentCloseTime, + static_cast(prevLedger.closeTime().time_since_epoch().count())); + doAcceptSpan.setAttribute( + cs::attr::closeTimeSelf, + static_cast(rawCloseTimes.self.time_since_epoch().count())); + doAcceptSpan.setAttribute( + cs::attr::closeTimeVoteBins, static_cast(rawCloseTimes.peers.size())); + doAcceptSpan.setAttribute( + cs::attr::disputesResolvedCount, static_cast(result.disputes.size())); + { + auto const prevRes = prevLedger.closeTimeResolution(); + auto const dir = [&]() -> std::string { + if (closeResolution > prevRes) + { + return "increased"; + } + if (closeResolution < prevRes) + { + return "decreased"; + } + return "unchanged"; + }(); + doAcceptSpan.setAttribute(cs::attr::resolutionDirection, std::move(dir)); + } + JLOG(j_.debug()) << "Report: Prop=" << (proposing ? "yes" : "no") << " val=" << (validating_ ? "yes" : "no") << " corLCL=" << (haveCorrectLCL ? "yes" : "no") @@ -503,12 +654,16 @@ RCLConsensus::Adaptor::doAccept( JLOG(j_.debug()) << "Building canonical tx set: " << retriableTxs.key(); + int64_t txCount = 0; for (auto const& item : *result.txns.map) { try { retriableTxs.insert(std::make_shared(SerialIter{item.slice()})); JLOG(j_.debug()) << " Tx: " << item.key(); + ++txCount; + auto const txHash = to_string(item.key()); + doAcceptSpan.addEvent(cs::event::txIncluded, {{cs::attr::txId, txHash}}); } catch (std::exception const& ex) { @@ -516,6 +671,7 @@ RCLConsensus::Adaptor::doAccept( JLOG(j_.warn()) << " Tx: " << item.key() << " throws: " << ex.what(); } } + doAcceptSpan.setAttribute(cs::attr::txCount, txCount); auto built = buildLCL( prevLedger, @@ -587,6 +743,11 @@ RCLConsensus::Adaptor::doAccept( // See if we can accept a ledger as fully-validated ledgerMaster_.consensusBuilt(built.ledger, result.txns.id(), std::move(consensusJson)); + // Record ledger close for OTel dashboard parity counter. Uses the + // call-site macro (see MetricMacros.h) rather than a MetricsRegistry + // member -- proof-of-concept for tasks/metric-macro-plan.md. + XRPL_METRIC_COUNTER_INC(app_, "ledgers_closed_total", "Total ledgers closed by consensus"); + //------------------------------------------------------------------------- { // Apply disputed transactions that didn't get in @@ -807,6 +968,19 @@ RCLConsensus::Adaptor::buildLCL( void RCLConsensus::Adaptor::validate(RCLCxLedger const& ledger, RCLTxSet const& txns, bool proposing) { + auto valSpan = createValidationSpan(); + if (valSpan) + { + namespace cs = telemetry::consensus::span; + valSpan->setAttribute(cs::attr::ledgerSeq, static_cast(ledger.seq())); + valSpan->setAttribute( + cs::attr::ledgerHash, to_string(ledger.ledger->header().hash).c_str()); + valSpan->setAttribute(cs::attr::proposing, proposing); + // proposing implies a full validation (vfFullValidation is set on + // the STValidation only when proposing — see below). + valSpan->setAttribute(cs::attr::fullValidation, proposing); + } + using namespace std::chrono_literals; auto validationTime = app_.getTimeKeeper().closeTime(); @@ -814,6 +988,13 @@ RCLConsensus::Adaptor::validate(RCLCxLedger const& ledger, RCLTxSet const& txns, validationTime = lastValidationTime_ + 1s; lastValidationTime_ = validationTime; + if (valSpan) + { + valSpan->setAttribute( + telemetry::consensus::span::attr::validationSignTime, + static_cast(validationTime.time_since_epoch().count())); + } + if (!validatorKeys_.keys) { JLOG(j_.warn()) << "RCLConsensus::Adaptor::validate: ValidatorKeys " @@ -885,15 +1066,42 @@ RCLConsensus::Adaptor::validate(RCLCxLedger const& ledger, RCLTxSet const& txns, // Broadcast to all our peers: protocol::TMValidation val; val.set_validation(serialized.data(), serialized.size()); + // Inject the current thread's active span context so receiving + // peers can link their validation.receive span as a child. + // + // The trace_context appended below is outside the signature on + // `serialized`, so it is not covered by validation authenticity. + // Downstream consumers treat it as advisory only. A signature-covered + // trace context is a possible future enhancement. + telemetry::SpanGuard::injectCurrentContextToProtobuf(*val.mutable_trace_context()); app_.getOverlay().broadcast(val); // Publish to all our subscribers: app_.getOPs().pubValidation(v); + + // Record validation sent for OTel dashboard parity counter. + if (auto* mr = app_.getMetricsRegistry()) + { + mr->incrementValidationsSent(); + // Record our validation for the agreement tracker so it can + // compare against network-validated ledgers. + mr->getValidationTracker().recordOurValidation(ledger.id(), ledger.seq()); + } } void RCLConsensus::Adaptor::onModeChange(ConsensusMode before, ConsensusMode after) { + namespace cs = telemetry::consensus::span; + + // Child of the round span via its captured context (roundSpan_ is a + // thread-free SpanGuard, so parent explicitly via its context). A mode + // change outside a round leaves roundSpanContext_ invalid, yielding a null + // guard (no-op). + auto span = telemetry::SpanGuard::childSpan(cs::modeChange, roundSpanContext_); + span.setAttribute(cs::attr::modeOld, toDisplayString(before).c_str()); + span.setAttribute(cs::attr::modeNew, toDisplayString(after).c_str()); + JLOG(j_.info()) << "Consensus mode change before=" << to_string(before) << ", after=" << to_string(after); @@ -1018,6 +1226,8 @@ RCLConsensus::Adaptor::preStartRound(RCLCxLedger const& prevLgr, hash_setsetAttribute(cs::attr::ledgerId, to_string(prevLgr.id()).c_str()); + roundSpan_->setAttribute(cs::attr::ledgerSeq, static_cast(prevLgr.seq()) + 1); + roundSpan_->setAttribute(cs::attr::mode, toDisplayString(mode_.load()).c_str()); + roundSpan_->setAttribute(cs::attr::traceStrategy, strategy.c_str()); + roundSpan_->setAttribute(cs::attr::roundId, static_cast(prevLgr.seq()) + 1); + roundSpan_->setAttribute(cs::attr::previousLedgerSeq, static_cast(prevLgr.seq())); + roundSpan_->setAttribute(cs::attr::previousProposers, static_cast(prevProposers_)); + roundSpan_->setAttribute( + cs::attr::previousRoundTimeMs, static_cast(prevRoundTime_.load().count())); + roundSpan_->setAttribute(cs::attr::consensusPhase, cs::val::phaseOpen); + + roundSpan_->addEvent(cs::event::phaseOpen); + + // roundSpanContext_ is the durable handle that child spans on other + // threads parent to. roundSpan_ is a thread-free SpanGuard that is + // reset() on a different worker than it was emplaced on, so spanContext() + // captures its own span and no scope work is needed. + roundSpanContext_ = roundSpan_->spanContext(); +} + +std::optional +RCLConsensus::Adaptor::createValidationSpan() +{ + namespace cs = telemetry::consensus::span; + + // Prefer linking to the accept span (matches the design diagram and + // the "validation follows acceptance" causal model). Fall back to the + // round span only if the accept context isn't yet captured (e.g. + // tracing started after onAccept, or makeAcceptSpan returned a null + // guard). + if (acceptSpanContext_.isValid()) + { + return telemetry::SpanGuard::linkedSpan(cs::validationSend, acceptSpanContext_); + } + + if (!roundSpanContext_.isValid()) + { + return std::nullopt; + } + + return telemetry::SpanGuard::linkedSpan(cs::validationSend, roundSpanContext_); +} + +void +RCLConsensus::Adaptor::onPhaseEvent(std::string_view eventName, std::string_view phaseLabel) +{ + namespace cs = telemetry::consensus::span; + + if (!roundSpan_ || !*roundSpan_) + { + return; + } + + roundSpan_->addEvent(eventName); + if (!phaseLabel.empty()) + { + roundSpan_->setAttribute(cs::attr::consensusPhase, phaseLabel); + } +} + +void +RCLConsensus::Adaptor::onOutcomeEvent(std::string_view eventName) +{ + if (!roundSpan_ || !*roundSpan_) + { + return; + } + roundSpan_->addEvent(eventName); +} + void RCLConsensus::startRound( NetClock::time_point const& now, diff --git a/src/xrpld/app/consensus/RCLConsensus.h b/src/xrpld/app/consensus/RCLConsensus.h index 4ffe18a7a89..c3bb1edea04 100644 --- a/src/xrpld/app/consensus/RCLConsensus.h +++ b/src/xrpld/app/consensus/RCLConsensus.h @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -34,6 +35,7 @@ #include #include #include +#include #include namespace xrpl { @@ -86,6 +88,60 @@ class RCLConsensus RCLCensorshipDetector censorshipDetector_; NegativeUNLVote nUnlVote_; + /** + * Span for the current consensus round. + * + * Created in preStartRound(), ended (via reset()) when the next + * round begins. When consensusTraceStrategy is "deterministic", + * the trace_id is derived from previousLedger.id() so that all + * validators in the same round share the same trace_id. + * + * Thread-free: a SpanGuard owns no thread-local Scope, so it can be + * emplaced and reset on different job workers safely. Child spans link + * via roundSpanContext_ (captured once), not an ambient parent span. + */ + std::optional roundSpan_; + + /** + * SpanContext snapshot of the current round span. + * + * Captured in startRoundTracing() as a lightweight value-type copy + * so that createValidationSpan() — which runs on the jtACCEPT + * worker thread — can build span links without accessing roundSpan_ + * across threads. + */ + telemetry::SpanContext roundSpanContext_; + + /** + * SpanContext of the prior round, captured before roundSpanContext_ + * is overwritten by the next round's startRoundTracing(). + * + * Used as a follows-from link target on the new round's span so + * consecutive rounds (each with its own deterministic trace-id) + * remain navigable in trace UIs as a connected sequence. + */ + telemetry::SpanContext prevRoundSpanContext_; + + /** + * SpanContext snapshot of the current round's accept span. + * + * Captured in makeAcceptSpan() and consumed by createValidationSpan() + * on the jtACCEPT worker thread so the validation.send span can be + * follows-from linked to consensus.accept (matching the design doc + * and span hierarchy diagram). Reset on each startRoundTracing() + * to prevent a stale prior-round context from being linked. + * + * Thread safety: same model as roundSpanContext_. The write in + * makeAcceptSpan happens on the main consensus thread under + * mutex_, and the read in createValidationSpan happens on the + * jtACCEPT worker thread that was posted by makeAcceptSpan. + * The job queue dispatch establishes a happens-before relation, + * so no atomic synchronization is needed. The next reset (in + * startRoundTracing) only runs after endConsensus → startRound, + * which by Consensus design only fires after doAccept completes. + */ + telemetry::SpanContext acceptSpanContext_; + public: using Ledger_t = RCLCxLedger; using NodeID_t = NodeID; @@ -178,6 +234,75 @@ class RCLConsensus return parms_; } + /** + * Set up the consensus round span and link it to the previous round. + * + * Saves the previous round's context for span-link construction, + * ends the old round span, and creates a new "consensus.round" span. + * Depending on the configured trace strategy the trace_id is either + * deterministic (derived from prevLgr hash) or random. + * + * @param prevLgr The ledger that will be the prior ledger for the + * new round. + */ + void + startRoundTracing(RCLCxLedger const& prevLgr); + + /** + * Create the "consensus.validation.send" span linked to the round. + * + * @return An engaged optional SpanGuard if tracing is active, + * std::nullopt otherwise. + */ + std::optional + createValidationSpan(); + + /** + * Record a phase-transition event on the active round span. + * + * Called from the engine at each phase boundary + * (open/establish/accepted/recovery) so the round span carries a + * complete timeline of state changes. Also updates the + * `consensus_phase` attribute to the current phase name. + * + * @param eventName Event name (e.g. "phase.establish"). + * @param phaseLabel String value for the consensus_phase attr + * ("open"/"establish"/"accepted"). Empty to skip + * the attribute update (e.g. for "recovery"). + */ + void + onPhaseEvent(std::string_view eventName, std::string_view phaseLabel); + + /** + * Record a checkConsensus outcome event on the round span. + * + * Called from the engine at the establish→accepted transition so + * the path that drove acceptance (Yes / MovedOn / Expired) is + * queryable from the round-level trace. + * + * @param eventName Event name (e.g. "outcome.yes"). + */ + void + onOutcomeEvent(std::string_view eventName); + + /** + * Captured context of the current round span. + * + * The generic engine uses this to parent the phase spans it owns + * (consensus.phase.open, consensus.establish) under the round span + * without touching roundSpan_ across threads. roundSpan_ is a + * thread-free SpanGuard (never an ambient parent), so child spans must + * link via this captured context. Returns an invalid context before the + * round span is created. + * + * @return The round span's captured context, or an invalid context. + */ + telemetry::SpanContext + roundSpanContext() const + { + return roundSpanContext_; + } + private: //--------------------------------------------------------------------- // The following members implement the generic Consensus requirements @@ -363,9 +488,19 @@ class RCLConsensus void notify(protocol::NodeEvent ne, RCLCxLedger const& ledger, bool haveCorrectLCL); + /** + * Create a consensus.accept span as a child of the round span. + * Returned via shared_ptr so it can be captured into the + * jtACCEPT lambda and live until doAccept completes. + */ + std::shared_ptr + makeAcceptSpan(Result const& result); + /** * Accept a new ledger based on the given transactions. * + * @param acceptSpan Parent span created by makeAcceptSpan(); + * accept.apply is created as its child. * @ref onAccept */ void @@ -375,7 +510,8 @@ class RCLConsensus NetClock::duration closeResolution, ConsensusCloseTimes const& rawCloseTimes, ConsensusMode const& mode, - json::Value&& consensusJson); + json::Value&& consensusJson, + std::shared_ptr acceptSpan); /** * Build the new last closed ledger. diff --git a/src/xrpld/app/ledger/InboundLedger.h b/src/xrpld/app/ledger/InboundLedger.h index d8a9ddf46b8..b7c61f9bf04 100644 --- a/src/xrpld/app/ledger/InboundLedger.h +++ b/src/xrpld/app/ledger/InboundLedger.h @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -193,6 +195,17 @@ class InboundLedger final : public TimeoutCounter, receivedData_; bool receiveDispatched_{false}; std::unique_ptr peerSet_; + + /** + * Spans the acquire lifecycle: started in init(), finalized in done() + * with the outcome (complete/failed), timeout count, and peer count. + * Gives operators visibility into back-fill / fork-recovery cost, which + * previously emitted no span or metric. + * Thread-free: emplaced by the acquiring thread, reset on a JtLedgerData + * worker. A SpanGuard owns no thread-local Scope, so it can be destroyed + * on the worker without corrupting the origin thread's context stack. + */ + std::optional acquireSpan_; }; } // namespace xrpl diff --git a/src/xrpld/app/ledger/LedgerHistory.cpp b/src/xrpld/app/ledger/LedgerHistory.cpp index 8734faa1fb2..d94f114d33c 100644 --- a/src/xrpld/app/ledger/LedgerHistory.cpp +++ b/src/xrpld/app/ledger/LedgerHistory.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -26,6 +27,7 @@ #include #include #include +#include #include #include @@ -323,11 +325,19 @@ LedgerHistory::handleMismatch( auto builtLedger = getLedgerByHash(built); auto validLedger = getLedgerByHash(valid); + // Records the classified mismatch reason as a labeled OTel counter so + // fork diagnosis is a queryable time series, not just a log grep. + auto recordReason = [this](std::string_view reason) { + if (auto* mr = app_.getMetricsRegistry()) + mr->incrementLedgerHistoryMismatch(reason); + }; + if (!builtLedger || !validLedger) { JLOG(j_.error()) << "MISMATCH cannot be analyzed:" << " builtLedger: " << to_string(built) << " -> " << builtLedger << " validLedger: " << to_string(valid) << " -> " << validLedger; + recordReason("unknown"); return; } @@ -349,6 +359,7 @@ LedgerHistory::handleMismatch( if (builtLedger->header().parentHash != validLedger->header().parentHash) { JLOG(j_.error()) << "MISMATCH on prior ledger"; + recordReason("prior_ledger"); return; } @@ -356,9 +367,16 @@ LedgerHistory::handleMismatch( if (builtLedger->header().closeTime != validLedger->header().closeTime) { JLOG(j_.error()) << "MISMATCH on close time"; + recordReason("close_time"); return; } + // Tracks whether the mismatch reason has already been recorded. The + // consensus tx-set hash disagreement is the root cause, so it is counted + // once here; the tx-level comparison below still logs its diagnostics but + // must not record a second reason for the same mismatch event. + bool reasonRecorded = false; + if (builtConsensusHash && validatedConsensusHash) { if (builtConsensusHash != validatedConsensusHash) @@ -366,6 +384,8 @@ LedgerHistory::handleMismatch( JLOG(j_.error()) << "MISMATCH on consensus transaction set " << " built: " << to_string(*builtConsensusHash) << " validated: " << to_string(*validatedConsensusHash); + recordReason("consensus_txset"); + reasonRecorded = true; } else { @@ -381,11 +401,15 @@ LedgerHistory::handleMismatch( if (builtTx == validTx) { JLOG(j_.error()) << "MISMATCH with same " << builtTx.size() << " transactions"; + if (!reasonRecorded) + recordReason("same_txset_diff_result"); } else { JLOG(j_.error()) << "MISMATCH with " << builtTx.size() << " built and " << validTx.size() << " valid transactions."; + if (!reasonRecorded) + recordReason("different_txset"); } JLOG(j_.error()) << "built\n" << getJson({*builtLedger, {}}); diff --git a/src/xrpld/app/ledger/detail/BuildLedger.cpp b/src/xrpld/app/ledger/detail/BuildLedger.cpp index e9c01c71339..482721bf6a7 100644 --- a/src/xrpld/app/ledger/detail/BuildLedger.cpp +++ b/src/xrpld/app/ledger/detail/BuildLedger.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -18,9 +19,13 @@ #include #include // IWYU pragma: keep #include +#include +#include #include +#include #include +#include #include #include #include @@ -44,6 +49,11 @@ buildLedgerImpl( beast::Journal j, ApplyTxs&& applyTxs) { + using namespace telemetry; + // Scoped so tx.apply (created synchronously below during applyTxs on this + // thread) nests under it. buildLedgerImpl runs synchronously with no yield. + auto buildSpan = ScopedSpanGuard(TraceCategory::Ledger, seg::ledger, ledger_span::op::build); + auto built = std::make_shared(*parent, closeTime); if (built->isFlagLedger()) @@ -77,6 +87,14 @@ buildLedgerImpl( built->header().seq < kXrpLedgerEarliestFees || built->read(keylet::feeSettings()), "xrpl::buildLedgerImpl : valid ledger fees"); built->setAccepted(closeTime, closeResolution, closeTimeCorrect); + buildSpan.setAttribute(ledger_span::attr::ledgerSeq, static_cast(built->header().seq)); + buildSpan.setAttribute( + ledger_span::attr::closeTime, static_cast(closeTime.time_since_epoch().count())); + buildSpan.setAttribute(ledger_span::attr::closeTimeCorrect, closeTimeCorrect); + buildSpan.setAttribute( + ledger_span::attr::closeResolutionMs, + static_cast( + std::chrono::duration_cast(closeResolution).count())); return built; } @@ -101,6 +119,9 @@ applyTransactions( OpenView& view, beast::Journal j) { + using namespace telemetry; + auto applySpan = SpanGuard::span(TraceCategory::Transactions, seg::tx, ledger_span::op::apply); + bool certainRetry = true; std::size_t count = 0; @@ -167,6 +188,8 @@ applyTransactions( // If there are any transactions left, we must have // tried them in at least one final pass XRPL_ASSERT(txns.empty() || !certainRetry, "xrpl::applyTransactions : retry transactions"); + applySpan.setAttribute(ledger_span::attr::txCount, static_cast(count)); + applySpan.setAttribute(ledger_span::attr::txFailed, static_cast(failed.size())); return count; } diff --git a/src/xrpld/app/ledger/detail/InboundLedger.cpp b/src/xrpld/app/ledger/detail/InboundLedger.cpp index 627a5d574f4..de2c3d6c27a 100644 --- a/src/xrpld/app/ledger/detail/InboundLedger.cpp +++ b/src/xrpld/app/ledger/detail/InboundLedger.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,8 @@ #include #include #include +#include +#include #include @@ -46,6 +49,7 @@ #include #include #include +#include #include #include #include @@ -95,6 +99,36 @@ InboundLedger::init(ScopedLockType& collectionLock) ScopedLockType sl(mtx_); collectionLock.unlock(); + // Span the acquire lifecycle so back-fill / fork-recovery cost is + // observable. Finalized in done() with the outcome and timeout count. + { + using namespace telemetry; + // acquireSpan_ is emplaced here but reset() on a JtLedgerData worker + // thread. A SpanGuard is thread-free (owns no thread-local Scope), so it + // can be created here and destroyed on the worker with no scope to strip. + acquireSpan_.emplace( + SpanGuard::span(TraceCategory::Ledger, seg::ledger, ledger_span::op::acquire)); + if (*acquireSpan_) + { + acquireSpan_->setAttribute(ledger_span::attr::ledgerSeq, static_cast(seq_)); + // Map the acquire reason to its span-attribute value. A switch + // keeps the mapping flat and exhaustive over the Reason enum. + std::string_view reasonVal = ledger_span::val::generic; + switch (reason_) + { + case Reason::HISTORY: + reasonVal = ledger_span::val::history; + break; + case Reason::CONSENSUS: + reasonVal = ledger_span::val::consensus; + break; + case Reason::GENERIC: + break; + } + acquireSpan_->setAttribute(ledger_span::attr::acquireReason, reasonVal); + } + } + tryDB(app_.getNodeFamily().db()); if (failed_) return; @@ -418,11 +452,37 @@ InboundLedger::done() signaled_ = true; touch(); - JLOG(journal_.debug()) << "Acquire " << hash_ << (failed_ ? " fail " : " ") - << ((timeouts_ == 0) - ? std::string() - : (std::string("timeouts:") + std::to_string(timeouts_) + " ")) - << stats_.get(); + // Finalize the acquire span with the outcome, timeout count, and peer + // count. Keep it active as the ambient context across the finalize and + // the outcome log below so that log line carries the span's trace_id. + // The activation is non-owning; acquireSpan_ still owns the span. The + // activation pops at the end of this block (restoring the prior context) + // while acquireSpan_ is still alive, then reset() ends the span. + { + auto acquireActivation = telemetry::activateIfLive(acquireSpan_); + if (acquireSpan_ && *acquireSpan_) + { + using namespace telemetry; + acquireSpan_->setAttribute( + ledger_span::attr::outcome, + failed_ ? std::string_view(ledger_span::val::failed) + : std::string_view(ledger_span::val::complete)); + acquireSpan_->setAttribute( + ledger_span::attr::timeouts, static_cast(timeouts_)); + acquireSpan_->setAttribute( + ledger_span::attr::peerCount, static_cast(getPeerCount())); + } + + JLOG(journal_.debug()) << "Acquire " << hash_ << (failed_ ? " fail " : " ") + << ((timeouts_ == 0) ? std::string() + : (std::string("timeouts:") + + std::to_string(timeouts_) + " ")) + << stats_.get(); + // acquireActivation pops here, before the span is ended below. + } + // End the acquire span after its outcome log. Unconditional so the span + // never leaks even when it was inactive. + acquireSpan_.reset(); XRPL_ASSERT(complete_ || failed_, "xrpl::InboundLedger::done : complete or failed"); diff --git a/src/xrpld/app/ledger/detail/LedgerMaster.cpp b/src/xrpld/app/ledger/detail/LedgerMaster.cpp index 2bd83b0f18b..bc5b517127c 100644 --- a/src/xrpld/app/ledger/detail/LedgerMaster.cpp +++ b/src/xrpld/app/ledger/detail/LedgerMaster.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include #include @@ -56,6 +58,8 @@ #include #include #include +#include +#include #include @@ -292,6 +296,11 @@ LedgerMaster::setValidLedger(std::shared_ptr const& l) (void)maxLedgerDifference_; validLedgerSeq_ = l->header().seq; + // Record the network-validated ledger for the agreement tracker so it + // can compare against our own validations. + if (auto* mr = app_.getMetricsRegistry()) + mr->getValidationTracker().recordNetworkValidation(l->header().hash, l->header().seq); + app_.getOPs().updateLocalTx(*l); app_.getSHAMapStore().onLedgerClosed(getValidatedLedger()); ledgerHistory_.validatedLedger(l, consensusHash); @@ -450,6 +459,10 @@ LedgerMaster::fixIndex(LedgerIndex ledgerIndex, LedgerHash const& ledgerHash) bool LedgerMaster::storeLedger(std::shared_ptr ledger) { + using namespace telemetry; + auto span = SpanGuard::span(TraceCategory::Ledger, seg::ledger, ledger_span::op::store); + span.setAttribute(ledger_span::attr::ledgerSeq, static_cast(ledger->header().seq)); + bool const validated = ledger->header().validated; // Returns true if we already had the ledger return ledgerHistory_.insert(ledger, validated); @@ -970,6 +983,11 @@ LedgerMaster::checkAccept(std::shared_ptr const& ledger) return; } + using namespace telemetry; + auto valSpan = SpanGuard::span(TraceCategory::Ledger, seg::ledger, ledger_span::op::validate); + valSpan.setAttribute(ledger_span::attr::ledgerSeq, static_cast(ledger->header().seq)); + valSpan.setAttribute(ledger_span::attr::validations, static_cast(tvc)); + JLOG(journal_.info()) << "Advancing accepted ledger to " << ledger->header().seq << " with >= " << minVal << " validations"; diff --git a/src/xrpld/app/ledger/detail/LedgerSpanNames.h b/src/xrpld/app/ledger/detail/LedgerSpanNames.h new file mode 100644 index 00000000000..4de4d248245 --- /dev/null +++ b/src/xrpld/app/ledger/detail/LedgerSpanNames.h @@ -0,0 +1,76 @@ +#pragma once + +/** + * Compile-time span name constants for ledger tracing. + * + * Used by BuildLedger and LedgerMaster for ledger lifecycle spans. + * Built on StaticStr/join() from SpanNames.h. + * + * Span hierarchy: + * + * ledger.build (BuildLedger — ledger construction) + * ledger.store (LedgerMaster — ledger storage) + * ledger.validate (LedgerMaster — ledger validation acceptance) + * ledger.acquire (InboundLedger — fetch a missing ledger from peers) + * tx.apply (BuildLedger — transaction application) + */ + +#include + +namespace xrpl::telemetry::ledger_span { + +// ===== Span operation suffixes =============================================== + +namespace op { +inline constexpr auto build = makeStr("build"); +inline constexpr auto store = makeStr("store"); +inline constexpr auto validate = makeStr("validate"); +inline constexpr auto apply = makeStr("apply"); +inline constexpr auto acquire = makeStr("acquire"); +} // namespace op + +// ===== Attribute keys ======================================================== + +namespace attr { +/** + * Canonical shared constants (defined in SpanNames.h). + */ +using ::xrpl::telemetry::attr::closeResolutionMs; +using ::xrpl::telemetry::attr::closeTime; +using ::xrpl::telemetry::attr::closeTimeCorrect; +using ::xrpl::telemetry::attr::ledgerHash; +using ::xrpl::telemetry::attr::ledgerSeq; + +/** + * Domain-owned bare attrs. + */ +inline constexpr auto txCount = makeStr("tx_count"); +inline constexpr auto txFailed = makeStr("tx_failed"); +inline constexpr auto validations = makeStr("validations"); + +/** + * ledger.acquire attrs (InboundLedger fetch lifecycle). + */ +inline constexpr auto acquireReason = makeStr("acquire_reason"); +inline constexpr auto timeouts = makeStr("timeouts"); +inline constexpr auto peerCount = makeStr("peer_count"); +inline constexpr auto outcome = makeStr("outcome"); +} // namespace attr + +// ===== Attribute values ====================================================== + +namespace val { +/** + * ledger.acquire outcome values. + */ +inline constexpr auto complete = makeStr("complete"); +inline constexpr auto failed = makeStr("failed"); +/** + * ledger.acquire reason values (mirror InboundLedger::Reason). + */ +inline constexpr auto history = makeStr("history"); +inline constexpr auto consensus = makeStr("consensus"); +inline constexpr auto generic = makeStr("generic"); +} // namespace val + +} // namespace xrpl::telemetry::ledger_span diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index d329475874b..3f083a932d0 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -81,10 +82,12 @@ #include #include // IWYU pragma: keep #include +#include #include #include #include // IWYU pragma: keep #include +#include #include #include #include @@ -98,6 +101,7 @@ #include #include #include +#include #include #include @@ -149,6 +153,7 @@ class ApplicationImp : public Application, public BasicApp beast::Journal journal_; beast::IOLatencyProbe probe_; std::atomic lastSample_; + std::atomic firstSample_; public: IOLatencySampler( @@ -156,7 +161,7 @@ class ApplicationImp : public Application, public BasicApp beast::Journal journal, std::chrono::milliseconds interval, boost::asio::io_context& ios) - : event_(std::move(ev)), journal_(journal), probe_(interval, ios) + : event_(std::move(ev)), journal_(journal), probe_(interval, ios), firstSample_(true) { } @@ -175,7 +180,10 @@ class ApplicationImp : public Application, public BasicApp lastSample_ = lastSample; - if (lastSample >= 10ms) + // Always emit the first sample so the metric is registered in + // downstream stores (Prometheus via StatsD). After that, only + // report latency >= 10 ms to avoid flooding with sub-ms values. + if (firstSample_.exchange(false) || lastSample >= 10ms) event_.notify(lastSample); if (lastSample >= 500ms) { @@ -212,6 +220,12 @@ class ApplicationImp : public Application, public BasicApp beast::Journal journal_; std::unique_ptr perfLog_; + std::unique_ptr telemetry_; + /** + * OTel metrics registry for gap-fill metrics (counters, histograms, + * observable gauges). Created after telemetry_ during setup(). + */ + std::unique_ptr metricsRegistry_; Application::MutexType masterMutex_; // Required by the SHAMapStore @@ -321,9 +335,24 @@ class ApplicationImp : public Application, public BasicApp *this, logs_->journal("PerfLog"), [this] { signalStop("PerfLog"); })) + , telemetry_( + telemetry::makeTelemetry( + telemetry::makeTelemetrySetup( + config_->section("telemetry"), + "", // Updated later via setServiceInstanceId() + BuildInfo::getVersionString(), + config_->networkId), + logs_->journal("Telemetry"))) + , txMaster_(*this) , collectorManager_(makeCollectorManager( config_->section(Sections::kInsight), + // Fall back to [telemetry] service_name so metrics inherit the + // same service.name as traces when [insight] omits it. Network + // type is derived from [network_id] via the shared telemetry + // helper, keeping metrics and traces on one network label. + config_->section("telemetry").valueOr("service_name", ""), + telemetry::networkTypeFromId(config_->networkId), logs_->journal("Collector"))) , jobQueue_( std::make_unique( @@ -367,7 +396,7 @@ class ApplicationImp : public Application, public BasicApp stopwatch(), logs_->journal("TaggedCache")) , cachedSLEs_( - "Cached SLEs", + "Cached_SLEs", 0, std::chrono::minutes(1), stopwatch(), @@ -649,6 +678,18 @@ class ApplicationImp : public Application, public BasicApp return *perfLog_; } + telemetry::Telemetry& + getTelemetry() override + { + return *telemetry_; + } + + telemetry::MetricsRegistry* + getMetricsRegistry() override + { + return metricsRegistry_.get(); + } + NodeCache& getTempNodeCache() override { @@ -1122,6 +1163,29 @@ class ApplicationImp : public Application, public BasicApp void startGenesisLedger(); + /** + * Start the tracing and metrics pipelines. + * + * Called once from setup(), just before the [rpc_startup] loop. Starting + * here (rather than in start()) guarantees the OTel MeterProvider is live + * before any metric-emitting code runs — including startup RPCs, whose + * PerfLog instrumentation records a call-site metric. A call-site metric + * macro caches its instrument on first use via std::call_once; if that + * first use happens while the meter is still empty, the instrument latches + * null for the process lifetime and the metric silently never records. + * + * The call site sits after overlay_ and the other subsystems are + * constructed, because the metrics reader thread starts here and its + * observable-gauge callbacks read that state (e.g. getOverlay()); starting + * earlier would let the reader observe a half-built application. + * + * @pre nodeIdentity_ is populated (needed for the service_instance_id + * fallback), metricsRegistry_ is constructed, and overlay_ (and the other + * subsystems read by observable-gauge callbacks) are constructed. + */ + void + startTelemetry(); + std::shared_ptr getLastFullLedger(); @@ -1297,6 +1361,20 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) nodeIdentity_ = getNodeIdentity(*this, cmdline); + // Now that the node identity is known, inject it into the telemetry + // resource attributes — but only if the user didn't already set a + // custom service_instance_id in [telemetry]. The Telemetry object + // was constructed with an empty serviceInstanceId because + // nodeIdentity_ is not available in the member initializer list. + if (!config_->section("telemetry").exists("service_instance_id")) + telemetry_->setServiceInstanceId(toBase58(TokenType::NodePublic, nodeIdentity_->first)); + + // Create the OTel MetricsRegistry for gap-fill metrics (counters, + // histograms, observable gauges). It is started later, just before the + // [rpc_startup] loop (see startTelemetry()). + metricsRegistry_ = std::make_unique( + telemetry_->isEnabled(), *this, logs_->journal("MetricsRegistry")); + if (!cluster_->load(config().section(Sections::kClusterNodes))) { JLOG(journal_.fatal()) << "Invalid entry in cluster configuration."; @@ -1438,6 +1516,16 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) JLOG(journal_.warn()) << "*** standalone signing solution as soon as possible."; } + // Start telemetry and metrics now — before the [rpc_startup] loop below — + // so the OTel meter is live before any metric-emitting code runs. Startup + // RPCs invoke PerfLog instrumentation that records a call-site metric; a + // metric macro caches its instrument on first use, so a first use before + // the meter exists would latch null for the process lifetime. Placed here, + // after overlay_ and the other subsystems the observable-gauge callbacks + // read are constructed, so the metrics reader thread never observes a + // half-built application. + startTelemetry(); + // // Execute start up rpc commands. // @@ -1511,6 +1599,31 @@ ApplicationImp::start(bool withTimers) perfLog_->start(); } +void +ApplicationImp::startTelemetry() +{ + // Start tracing first so subsequent startup/early activity can be traced. + telemetry_->start(); + + // Start the metrics pipeline after telemetry; the endpoint uses the + // same base URL but the /v1/metrics path. + if (metricsRegistry_) + { + auto const& section = config_->section("telemetry"); + std::string endpoint = "http://localhost:4318/v1/metrics"; + set(endpoint, "metrics_endpoint", section); + + // Pass the service_instance_id so the MeterProvider Resource + // carries it, giving Prometheus an service_instance_id label. + std::string instanceId; + set(instanceId, "service_instance_id", section); + if (instanceId.empty() && nodeIdentity_) + instanceId = toBase58(TokenType::NodePublic, nodeIdentity_->first); + + metricsRegistry_->start(endpoint, instanceId); + } +} + void ApplicationImp::run() { @@ -1586,6 +1699,19 @@ ApplicationImp::run() // The order of these stop calls is delicate. // Re-ordering them risks undefined behavior. loadManager_->stop(); + + // Detach MetricsRegistry observable-gauge callbacks BEFORE stopping + // any service the callbacks read from. The callbacks run on the OTel + // reader thread and touch nodeStore_, overlay_, networkOPs_, + // ledgerMaster, inboundLedgers, etc. A final tick that fires after + // one of those services has shut down would dereference dangling + // state. detachCallbacks() flips an atomic flag every callback + // acquire-loads at its entry, so subsequent ticks become no-ops. + // The final provider teardown still happens in metricsRegistry_->stop() + // farther down. + if (metricsRegistry_) + metricsRegistry_->detachCallbacks(); + shaMapStore_->stop(); jobQueue_->stop(); if (overlay_) @@ -1599,6 +1725,15 @@ ApplicationImp::run() ledgerCleaner_->stop(); nodeStore_->stop(); perfLog_->stop(); + // Stop metrics pipeline before telemetry — gauge callbacks reference + // Application services that may be shutting down. + if (metricsRegistry_) + metricsRegistry_->stop(); + // Telemetry must stop last among trace-producing components. + // serverHandler_, overlay_, and jobQueue_ are already stopped above, + // so no threads should be calling startSpan() at this point. + // See TODO in TelemetryImpl::stop() re: thread-safety of sdkProvider_. + telemetry_->stop(); JLOG(journal_.info()) << "Done."; } diff --git a/src/xrpld/app/main/CollectorManager.cpp b/src/xrpld/app/main/CollectorManager.cpp index 9e1278607f6..84dcddb6698 100644 --- a/src/xrpld/app/main/CollectorManager.cpp +++ b/src/xrpld/app/main/CollectorManager.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -24,7 +25,12 @@ class CollectorManagerImp : public CollectorManager std::unique_ptr groups_; // NOLINTEND(readability-identifier-naming) - CollectorManagerImp(Section const& params, beast::Journal journal) : journal_(journal) + CollectorManagerImp( + Section const& params, + std::string const& serviceName, + std::string const& networkType, + beast::Journal journal) + : journal_(journal) { std::string const& server = get(params, Keys::kServer); @@ -36,6 +42,33 @@ class CollectorManagerImp : public CollectorManager collector_ = beast::insight::StatsDCollector::make(address, prefix, journal); } + // LCOV_EXCL_START -- OTel collector path is not exercised in unit tests + else if (server == "otel") + { + // Read OTLP metrics endpoint from [insight] section. + // Default to the standard OTLP/HTTP metrics path on localhost. + std::string endpoint = get(params, "endpoint"); + if (endpoint.empty()) + endpoint = "http://localhost:4318/v1/metrics"; + std::string const& prefix(get(params, "prefix")); + + // Read service_instance_id, same key as the [telemetry] + // section uses, so multi-node deployments can distinguish + // metric sources via the service_instance_id Prometheus label. + std::string const instanceId = get(params, "service_instance_id"); + + // service.name from [insight] (falls back to the value the + // caller derived from [telemetry]); network type derived from + // [network_id]. Both mirror the trace exporter so metrics and + // traces carry the same service and network identity. + std::string serviceNameCfg = get(params, "service_name"); + if (serviceNameCfg.empty()) + serviceNameCfg = serviceName; + + collector_ = beast::insight::OTelCollector::New( + endpoint, prefix, instanceId, serviceNameCfg, networkType, journal); + } + // LCOV_EXCL_STOP else { collector_ = beast::insight::NullCollector::make(); @@ -62,9 +95,13 @@ class CollectorManagerImp : public CollectorManager //------------------------------------------------------------------------------ std::unique_ptr -makeCollectorManager(Section const& params, beast::Journal journal) +makeCollectorManager( + Section const& params, + std::string const& serviceName, + std::string const& networkType, + beast::Journal journal) { - return std::make_unique(params, journal); + return std::make_unique(params, serviceName, networkType, journal); } } // namespace xrpl diff --git a/src/xrpld/app/main/CollectorManager.h b/src/xrpld/app/main/CollectorManager.h index fac72b4de1e..2539b7ecc8a 100644 --- a/src/xrpld/app/main/CollectorManager.h +++ b/src/xrpld/app/main/CollectorManager.h @@ -25,7 +25,21 @@ class CollectorManager group(std::string const& name) = 0; }; +/** + * Construct the collector manager. + * + * @param params The [insight] config section. + * @param serviceName service.name resource attribute for OTel metrics + * (empty -> the collector defaults it to "xrpld"). + * @param networkType xrpl.network.type resource attribute for OTel + * metrics (e.g. "mainnet"), derived from [network_id]. + * @param journal Journal for logging. + */ std::unique_ptr -makeCollectorManager(Section const& params, beast::Journal journal); +makeCollectorManager( + Section const& params, + std::string const& serviceName, + std::string const& networkType, + beast::Journal journal); } // namespace xrpl diff --git a/src/xrpld/app/main/GRPCServer.cpp b/src/xrpld/app/main/GRPCServer.cpp index 1146cbdc08c..566362bb0f6 100644 --- a/src/xrpld/app/main/GRPCServer.cpp +++ b/src/xrpld/app/main/GRPCServer.cpp @@ -2,6 +2,7 @@ #include // IWYU pragma: keep #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include @@ -49,6 +51,7 @@ #include #include #include +#include #include #include @@ -94,7 +97,8 @@ GRPCServerImpl::CallData::CallData( Forward forward, RPC::Condition requiredCondition, Resource::Charge loadType, - std::vector const& secureGatewayIPs) + std::vector const& secureGatewayIPs, + std::string_view name) : service_(service) , cq_(cq) , finished_(false) @@ -106,6 +110,7 @@ GRPCServerImpl::CallData::CallData( , requiredCondition_(requiredCondition) , loadType_(std::move(loadType)) , secureGatewayIPs_(secureGatewayIPs) + , name_(name) { // Bind a listener. When a request is received, "this" will be returned // from CompletionQueue::Next @@ -125,7 +130,8 @@ GRPCServerImpl::CallData::clone() forward_, requiredCondition_, loadType_, - secureGatewayIPs_); + secureGatewayIPs_, + name_); } template @@ -163,12 +169,19 @@ template void GRPCServerImpl::CallData::process(std::shared_ptr coro) { + using namespace telemetry; + auto span = SpanGuard::span(TraceCategory::Rpc, grpc_span::prefix::grpc, name_); + span.setAttribute(grpc_span::attr::method, name_); + try { auto usage = getUsage(); bool const isUnlimited = clientIsUnlimited(); if (!isUnlimited && usage.disconnect(app_.getJournal("gRPCServer"))) { + span.setAttribute( + grpc_span::attr::grpcStatus, grpc_span::val::error); // LCOV_EXCL_LINE + span.setError(grpc_span::val::resourceExhausted); // LCOV_EXCL_LINE grpc::Status const status{ grpc::StatusCode::RESOURCE_EXHAUSTED, "usage balance exceeds threshold"}; responder_.FinishWithError(status, this); @@ -179,6 +192,11 @@ GRPCServerImpl::CallData::process(std::shared_ptr #include #include +#include #include #include #include @@ -196,6 +197,11 @@ class GRPCServerImpl final std::vector const& secureGatewayIPs_; + /** + * Human-readable name for telemetry spans (e.g. "GetLedger"). + */ + std::string_view name_; + public: ~CallData() override = default; @@ -211,7 +217,8 @@ class GRPCServerImpl final Forward forward, RPC::Condition requiredCondition, Resource::Charge loadType, - std::vector const& secureGatewayIPs); + std::vector const& secureGatewayIPs, + std::string_view name = ""); CallData(CallData const&) = delete; diff --git a/src/xrpld/app/main/GrpcSpanNames.h b/src/xrpld/app/main/GrpcSpanNames.h new file mode 100644 index 00000000000..2eb2f44ca2f --- /dev/null +++ b/src/xrpld/app/main/GrpcSpanNames.h @@ -0,0 +1,66 @@ +#pragma once + +/** + * Compile-time span name constants for the gRPC subsystem. + * + * All span prefixes, operation names, and attribute keys used by gRPC + * tracing call sites are defined here. Built on the StaticStr/join() + * primitives from . + * + * Span hierarchy: + * + * +-------------------------------------------------------+ + * | grpc. (e.g. grpc.GetLedger) | + * | CallData::process(coro) | + * | attrs: method, grpc_role, grpc_status | + * +-------------------------------------------------------+ + * + * Unlike the HTTP/WS RPC path, gRPC has a flat single-span structure + * per request since each CallData handles exactly one RPC method. + * The method name is embedded in the span name (rather than only as + * an attribute) so dashboards can break out per-method latency and + * error rates without needing TraceQL attribute filters. + */ + +#include + +namespace xrpl::telemetry::grpc_span { + +// ===== Span prefixes ======================================================= + +namespace prefix { +/** + * "grpc" — root prefix for gRPC transport spans. The full span name is + * formed at the call site as `grpc.` (see GRPCServer.cpp). + */ +inline constexpr auto grpc = makeStr("grpc"); +} // namespace prefix + +// ===== Attribute keys ====================================================== + +namespace attr { +/** + * "method" — gRPC method name (e.g. GetLedger). + */ +inline constexpr auto method = makeStr("method"); +/** + * "grpc_role" — Domain-qualified: collides with rpc_role. + */ +inline constexpr auto grpcRole = makeStr("grpc_role"); +/** + * "grpc_status" — Domain-qualified: avoids OTel reserved span status. + */ +inline constexpr auto grpcStatus = makeStr("grpc_status"); +} // namespace attr + +// ===== Attribute values ==================================================== + +namespace val { +using telemetry::attr_val::error; +using telemetry::attr_val::success; +inline constexpr auto admin = makeStr("admin"); +inline constexpr auto user = makeStr("user"); +inline constexpr auto resourceExhausted = makeStr("resource_exhausted"); +} // namespace val + +} // namespace xrpl::telemetry::grpc_span diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index 4b0091dff6f..b3a2e75c4ae 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -33,6 +33,10 @@ #include #include #include +#include +#include +#include +#include #include #include @@ -114,6 +118,7 @@ #include #include #include +#include #include #include @@ -169,9 +174,17 @@ class NetworkOPsImp final : public NetworkOPs FailHard const failType; bool applied = false; TER result; - - TransactionStatus(std::shared_ptr t, bool a, bool l, FailHard f) - : transaction(std::move(t)), admin(a), local(l), failType(f) + std::shared_ptr span; ///< Keeps the tx.process + ///< span alive until the + ///< batch processes this. + + TransactionStatus( + std::shared_ptr t, + bool a, + bool l, + FailHard f, + std::shared_ptr s = nullptr) + : transaction(std::move(t)), admin(a), local(l), failType(f), span(std::move(s)) { XRPL_ASSERT( local || failType == FailHard::No, @@ -264,6 +277,21 @@ class NetworkOPsImp final : public NetworkOPs .start = start_, .initialSyncUs = initialSyncUs_}; } + + /** + * Time spent in the current operating mode so far. This is the same + * quantity reported as `server_state_duration_us` in json(): the + * elapsed time since the last state transition. Thread-safe. + * + * @return Duration since entering the current state, in microseconds. + */ + std::chrono::microseconds + currentStateDurationUs() const + { + std::scoped_lock const lock(mutex_); + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_); + } }; /** @@ -348,6 +376,9 @@ class NetworkOPsImp final : public NetworkOPs OperatingMode getOperatingMode() const override; + std::chrono::microseconds + getServerStateDurationUs() const override; + std::string strOperatingMode(OperatingMode const mode, bool const admin) const override; @@ -379,9 +410,15 @@ class NetworkOPsImp final : public NetworkOPs * @param transaction Transaction object. * @param bUnlimited Whether a privileged client connection submitted it. * @param failType fail_hard setting from transaction submission. + * @param span Optional tx.process span to keep alive across the + * batch boundary so its context propagates to peers. */ void - doTransactionSync(std::shared_ptr transaction, bool bUnlimited, FailHard failType); + doTransactionSync( + std::shared_ptr transaction, + bool bUnlimited, + FailHard failType, + std::shared_ptr span = nullptr); /** * For transactions not submitted by a locally connected client, fire and @@ -396,7 +433,8 @@ class NetworkOPsImp final : public NetworkOPs doTransactionAsync( std::shared_ptr transaction, bool bUnlimited, - FailHard failtype); + FailHard failtype, + std::shared_ptr span = nullptr); private: bool @@ -966,6 +1004,12 @@ NetworkOPsImp::getOperatingMode() const return mode_; } +std::chrono::microseconds +NetworkOPsImp::getServerStateDurationUs() const +{ + return accounting_.currentStateDurationUs(); +} + inline std::string NetworkOPsImp::strOperatingMode(bool const admin /* = false */) const { @@ -1364,6 +1408,29 @@ NetworkOPsImp::processTransaction( bool bLocal, FailHard failType) { + using namespace telemetry; + // SpanGuard is thread-free (holds no Scope), so it is safe to store here + // and end on the batch worker thread that later applies this transaction — + // no detach step is needed. + auto span = std::make_shared(txProcessSpan(transaction->getID())); + span->setAttribute(tx_span::attr::txHash, to_string(transaction->getID()).c_str()); + span->setAttribute(tx_span::attr::local, bLocal); + // The current (open) ledger index at submission/relay time — the ledger + // being worked on. Correlates this tx.process to the ledger trace; the tx + // has not yet been applied to a specific ledger here, so there is no hash. + span->setAttribute( + tx_span::attr::currentLedgerSeq, + static_cast(ledgerMaster_.getCurrentLedgerIndex())); + if (auto const& stx = transaction->getSTransaction()) + { + if (auto const* fmt = TxFormats::getInstance().findByType(stx->getTxnType())) + span->setAttribute(tx_span::attr::txType, fmt->getName().c_str()); + span->setAttribute( + tx_span::attr::fee, static_cast(stx->getFieldAmount(sfFee).xrp().drops())); + span->setAttribute( + tx_span::attr::sequence, static_cast(stx->getSeqProxy().value())); + } + auto ev = jobQueue_.makeLoadEvent(JtTxnProc, "ProcessTXN"); // preProcessTransaction can change our pointer @@ -1372,11 +1439,13 @@ NetworkOPsImp::processTransaction( if (bLocal) { - doTransactionSync(transaction, bUnlimited, failType); + span->setAttribute(tx_span::attr::path, tx_span::val::sync); + doTransactionSync(transaction, bUnlimited, failType, std::move(span)); } else { - doTransactionAsync(transaction, bUnlimited, failType); + span->setAttribute(tx_span::attr::path, tx_span::val::async); + doTransactionAsync(transaction, bUnlimited, failType, std::move(span)); } } @@ -1384,14 +1453,15 @@ void NetworkOPsImp::doTransactionAsync( std::shared_ptr transaction, bool bUnlimited, - FailHard failType) + FailHard failType, + std::shared_ptr span) { std::scoped_lock const lock(mutex_); if (transaction->getApplying()) return; - transactions_.emplace_back(transaction, bUnlimited, false, failType); + transactions_.emplace_back(transaction, bUnlimited, false, failType, std::move(span)); transaction->setApplying(); if (dispatchState_ == DispatchState::None) @@ -1407,13 +1477,14 @@ void NetworkOPsImp::doTransactionSync( std::shared_ptr transaction, bool bUnlimited, - FailHard failType) + FailHard failType, + std::shared_ptr span) { std::unique_lock lock(mutex_); if (!transaction->getApplying()) { - transactions_.emplace_back(transaction, bUnlimited, true, failType); + transactions_.emplace_back(transaction, bUnlimited, true, failType, std::move(span)); transaction->setApplying(); } @@ -1560,8 +1631,19 @@ NetworkOPsImp::apply(std::unique_lock& batchLock) if (e.failType == FailHard::Yes) flags |= TapFailHard; + // Parent the txq.enqueue span to this tx's tx.process span + // via an explicit captured context (the parent is explicit, + // not ambient-inherited). Null on the open-ledger rebuild + // path, where no tx.process span exists. + auto const txProcessCtx = + (e.span && *e.span) ? e.span->spanContext() : telemetry::SpanContext{}; auto const result = registry_.get().getTxQ().apply( - registry_.get().getApp(), view, e.transaction->getSTransaction(), flags, j); + registry_.get().getApp(), + view, + e.transaction->getSTransaction(), + flags, + j, + txProcessCtx.isValid() ? &txProcessCtx : nullptr); e.result = result.ter; e.applied = result.applied; changed = changed || result.applied; @@ -1579,6 +1661,19 @@ NetworkOPsImp::apply(std::unique_lock& batchLock) auto newOL = registry_.get().getOpenLedger().current(); for (TransactionStatus const& e : transactions) { + // Make this transaction's span ambient for the duration of its + // apply so the per-tx log lines below carry its trace_id. + // Non-owning: the span is still owned/ended by e.span. Scoped to + // one loop iteration, so each tx's span is ambient only for its + // own processing. No yield in this loop. + auto txActivation = telemetry::activateIfLive(e.span); + + if (e.span && *e.span) + { + e.span->setAttribute( + telemetry::tx_span::attr::terResult, transToken(e.result).c_str()); + e.span->setAttribute(telemetry::tx_span::attr::applied, e.applied); + } e.transaction->clearSubmitResult(); if (e.applied) @@ -1739,6 +1834,10 @@ NetworkOPsImp::apply(std::unique_lock& batchLock) tx.set_receivetimestamp( registry_.get().getTimeKeeper().now().time_since_epoch().count()); tx.set_deferred(e.result == terQUEUED); + // Inject the tx.process span's trace context so the + // receiving node can link its tx.receive span as a child. + if (e.span && *e.span) + telemetry::injectSpanContext(*e.span, *tx.mutable_trace_context()); // FIXME: This should be when we received it registry_.get().getOverlay().relay(e.transaction->getID(), tx, *toSkip); e.transaction->setBroadcast(); @@ -2566,6 +2665,10 @@ NetworkOPsImp::setMode(OperatingMode om) accounting_.mode(om); + // Record state change for OTel dashboard parity counter. + if (auto* mr = registry_.get().getMetricsRegistry()) + mr->incrementStateChanges(); + JLOG(journal_.info()) << "STATE->" << strOperatingMode(); pubServer(); } @@ -2574,6 +2677,8 @@ bool NetworkOPsImp::recvValidation(std::shared_ptr const& val, std::string const& source) { JLOG(journal_.trace()) << "recvValidation " << val->getLedgerHash() << " from " << source; + if (auto* mr = registry_.get().getMetricsRegistry()) + mr->incrementValidationsChecked(); std::unique_lock lock(validationsMutex_); BypassAccept bypassAccept = BypassAccept::No; diff --git a/src/xrpld/app/misc/TxQ.h b/src/xrpld/app/misc/TxQ.h index 65b4e9778c4..bbcbffecf32 100644 --- a/src/xrpld/app/misc/TxQ.h +++ b/src/xrpld/app/misc/TxQ.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -334,7 +335,8 @@ class TxQ OpenView& view, std::shared_ptr const& tx, ApplyFlags flags, - beast::Journal j); + beast::Journal j, + telemetry::SpanContext const* parentCtx = nullptr); /** * Fill the new open ledger with transactions from the queue. diff --git a/src/xrpld/app/misc/detail/TxQ.cpp b/src/xrpld/app/misc/detail/TxQ.cpp index 041d2ade1e5..35c64ba7f9d 100644 --- a/src/xrpld/app/misc/detail/TxQ.cpp +++ b/src/xrpld/app/misc/detail/TxQ.cpp @@ -2,8 +2,11 @@ #include #include +#include +#include #include +#include #include #include #include @@ -30,6 +33,7 @@ #include #include #include +#include #include #include @@ -541,6 +545,10 @@ TxQ::tryClearAccountQueueUpThruTx( FeeMetrics::Snapshot const& metricsSnapshot, beast::Journal j) { + using namespace telemetry; + [[maybe_unused]] ScopedSpanGuard span( + TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::batchClear); + SeqProxy const tSeqProx{tx.getSeqProxy()}; XRPL_ASSERT( beginTxIter != accountIter->second.transactions.end(), @@ -612,7 +620,8 @@ TxQ::tryClearAccountQueueUpThruTx( if (txResult.applied) { // All of the queued transactions applied, so remove them from the - // queue. + // queue. `dist` queued txs preceded the current one in the batch. + span.setAttribute(txq_span::attr::numCleared, static_cast(dist)); endTxIter = erase(accountIter->second, beginTxIter, endTxIter); // If `tx` is replacing a queued tx, delete that one, too. if (endTxIter != accountIter->second.transactions.end() && endTxIter->first == tSeqProx) @@ -741,8 +750,39 @@ TxQ::apply( OpenView& view, std::shared_ptr const& tx, ApplyFlags flags, - beast::Journal j) + beast::Journal j, + telemetry::SpanContext const* parentCtx) { + using namespace telemetry; + // Parent the enqueue span to the caller's tx.process span when a context is + // supplied (submission path), using the explicit-context factory: the parent + // comes from the captured ctx rather than being inherited from whatever span + // is ambient, and this ScopedSpanGuard's scope is RAII-bounded to this fully + // synchronous call (no coroutine yield), so no unrelated parent leaks in and + // its scope cannot leak out onto a reused worker. + // On the open-ledger rebuild path parentCtx is null and this is a root; the + // current_ledger_seq attribute below correlates it to the ledger instead. + // A lambda (not a ternary) picks the factory: ScopedSpanGuard's move ctor is + // deleted, so guaranteed copy elision on each return is the only way to + // construct it conditionally. + auto span = [&]() -> ScopedSpanGuard { + if (parentCtx && parentCtx->isValid()) + return ScopedSpanGuard::childSpan(txq_span::enqueue, *parentCtx); + return ScopedSpanGuard( + TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::enqueue); + }(); + span.setAttribute(txq_span::attr::txHash, to_string(tx->getTransactionID()).c_str()); + if (auto const* fmt = TxFormats::getInstance().findByType(tx->getTxnType())) + span.setAttribute(txq_span::attr::txType, fmt->getName().c_str()); + // The ledger being worked on (open/tentative apply or in-flight consensus + // build) — correlates this enqueue to the ledger trace in every context. + span.setAttribute(txq_span::attr::currentLedgerSeq, static_cast(view.seq())); + span.setAttribute( + txq_span::attr::currentLedgerHash, to_string(view.header().parentHash).c_str()); + // Default outcome; overridden below on the direct-apply and queued paths. + // Every other early return leaves the tx rejected from the queue. + span.setAttribute(txq_span::attr::txqStatus, txq_span::val::rejected); + // See if the transaction is valid, properly formed, // etc. before doing potentially expensive queue // replace and multi-transaction operations. @@ -753,7 +793,10 @@ TxQ::apply( // See if the transaction paid a high enough fee that it can go straight // into the ledger. if (auto directApplied = tryDirectApply(app, view, tx, flags, j)) + { + span.setAttribute(txq_span::attr::txqStatus, txq_span::val::appliedDirect); return *directApplied; + } if ((flags & TapDryRun) != 0u) return {telCAN_NOT_QUEUE, false}; @@ -880,6 +923,10 @@ TxQ::apply( auto const metricsSnapshot = feeMetrics_.getSnapshot(); auto const feeLevelPaid = getFeeLevelPaid(view, *tx); auto const requiredFeeLevel = getRequiredFeeLevel(view, flags, metricsSnapshot, lock); + span.setAttribute( + txq_span::attr::feeLevelPaid, static_cast(feeLevelPaid.value())); + span.setAttribute( + txq_span::attr::requiredFeeLevel, static_cast(requiredFeeLevel.value())); // Is there a blocker already in the account's queue? If so, don't // allow additional transactions in the queue. @@ -1213,6 +1260,7 @@ TxQ::apply( /* Can't erase (*replacedTxIter) here because success implies that it has already been deleted. */ + span.setAttribute(txq_span::attr::txqStatus, txq_span::val::applied); return result; } } @@ -1250,6 +1298,8 @@ TxQ::apply( JLOG(j_.info()) << "Queue is full, and transaction " << transactionID << " would kick a transaction from the same account (" << account << ") out of the queue."; + if (auto* const metrics = app.getMetricsRegistry(); metrics != nullptr) + metrics->incrementTxqDropped("queue_full"); return {telCAN_NOT_QUEUE_FULL, false}; } auto const& endAccount = byAccount_.at(lastRIter->account); @@ -1293,6 +1343,8 @@ TxQ::apply( { JLOG(j_.info()) << "Queue is full, and transaction " << transactionID << " fee is lower than end item's account average fee"; + if (auto* const metrics = app.getMetricsRegistry(); metrics != nullptr) + metrics->incrementTxqDropped("queue_full"); return {telCAN_NOT_QUEUE_FULL, false}; } } @@ -1328,6 +1380,7 @@ TxQ::apply( << " to queue." << " Flags: " << flags; + span.setAttribute(txq_span::attr::txqStatus, txq_span::val::queued); return {terQUEUED, false}; } @@ -1346,6 +1399,10 @@ TxQ::apply( void TxQ::processClosedLedger(Application& app, ReadView const& view, bool timeLeap) { + using namespace telemetry; + ScopedSpanGuard span(TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::cleanup); + span.setAttribute(txq_span::attr::ledgerSeq, static_cast(view.header().seq)); + std::scoped_lock const lock(mutex_); feeMetrics_.update(app, view, timeLeap, setup_); @@ -1357,18 +1414,26 @@ TxQ::processClosedLedger(Application& app, ReadView const& view, bool timeLeap) maxSize_ = std::max(snapshot.txnsExpected * setup_.ledgersInQueue, setup_.queueSizeMin); // Remove any queued candidates whose LastLedgerSequence has gone by. + auto* const metrics = app.getMetricsRegistry(); + std::int64_t expiredCount = 0; for (auto candidateIter = byFee_.begin(); candidateIter != byFee_.end();) { if (candidateIter->lastValid && *candidateIter->lastValid <= ledgerSeq) { byAccount_.at(candidateIter->account).dropPenalty = true; candidateIter = erase(candidateIter); + // Count each expired transaction: submitters who under-bid the + // escalating fee and were never included before expiry. + if (metrics != nullptr) + metrics->incrementTxqExpired(); + ++expiredCount; } else { ++candidateIter; } } + span.setAttribute(txq_span::attr::expiredCount, expiredCount); // Remove any TxQAccounts that don't have candidates // under them @@ -1417,6 +1482,8 @@ TxQ::processClosedLedger(Application& app, ReadView const& view, bool timeLeap) bool TxQ::accept(Application& app, OpenView& view) { + using namespace telemetry; + /* Move transactions from the queue from largest fee level to smallest. As we add more transactions, the required fee level will increase. Stop when the transaction fee level gets lower than the required fee @@ -1427,6 +1494,11 @@ TxQ::accept(Application& app, OpenView& view) std::scoped_lock const lock(mutex_); + // Create the span and read byFee_.size() only after taking the lock, since + // byFee_ is guarded by mutex_. + ScopedSpanGuard span(TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::accept); + span.setAttribute(txq_span::attr::queueSize, static_cast(byFee_.size())); + auto const metricsSnapshot = feeMetrics_.getSnapshot(); for (auto candidateIter = byFee_.begin(); candidateIter != byFee_.end();) @@ -1454,10 +1526,19 @@ TxQ::accept(Application& app, OpenView& view) JLOG(j_.trace()) << "Applying queued transaction " << candidateIter->txID << " to open ledger."; + ScopedSpanGuard txSpan( + TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::acceptTx); + txSpan.setAttribute(txq_span::attr::txHash, to_string(candidateIter->txID).c_str()); + txSpan.setAttribute( + txq_span::attr::retriesRemaining, + static_cast(candidateIter->retriesRemaining)); + auto const [txnResult, didApply, _metadata] = candidateIter->apply(app, view, j_); + txSpan.setAttribute(txq_span::attr::terCode, transToken(txnResult).c_str()); if (didApply) { + txSpan.setAttribute(txq_span::attr::txqStatus, txq_span::val::applied); // Remove the candidate from the queue JLOG(j_.debug()) << "Queued transaction " << candidateIter->txID << " applied successfully with " << transToken(txnResult) @@ -1478,12 +1559,14 @@ TxQ::accept(Application& app, OpenView& view) { account.dropPenalty = true; } + txSpan.setAttribute(txq_span::attr::txqStatus, txq_span::val::failed); JLOG(j_.debug()) << "Queued transaction " << candidateIter->txID << " failed with " << transToken(txnResult) << ". Remove from queue."; candidateIter = eraseAndAdvance(candidateIter); } else { + txSpan.setAttribute(txq_span::attr::txqStatus, txq_span::val::retried); JLOG(j_.debug()) << "Queued transaction " << candidateIter->txID << " failed with " << transToken(txnResult) << ". Leave in queue." << " Applied: " << didApply << ". Flags: " << candidateIter->flags; @@ -1579,6 +1662,7 @@ TxQ::accept(Application& app, OpenView& view) } } XRPL_ASSERT(byFee_.size() == startingSize, "xrpl::TxQ::accept : byFee size match"); + span.setAttribute(txq_span::attr::ledgerChanged, ledgerChanged); return ledgerChanged; } @@ -1662,6 +1746,10 @@ TxQ::tryDirectApply( ApplyFlags flags, beast::Journal j) { + using namespace telemetry; + [[maybe_unused]] ScopedSpanGuard const span( + TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::applyDirect); + auto const account = (*tx)[sfAccount]; auto const sleAccount = view.read(keylet::account(account)); diff --git a/src/xrpld/app/misc/detail/TxQSpanNames.h b/src/xrpld/app/misc/detail/TxQSpanNames.h new file mode 100644 index 00000000000..6b64b76a575 --- /dev/null +++ b/src/xrpld/app/misc/detail/TxQSpanNames.h @@ -0,0 +1,148 @@ +#pragma once + +/** + * Compile-time span name constants for Transaction Queue tracing. + * + * Covers the TxQ lifecycle: enqueue decisions, direct apply, batch + * clear, ledger-close accept loop, per-tx apply, and cleanup. + * + * Span hierarchy: + * + * Transaction submission: + * + * +-------------------------------------------------------+ + * | tx.process (existing, from TxSpanNames.h) | + * | | + * | +--------------------------------------------------+ | + * | | txq.enqueue | | + * | | TxQ::apply() | | + * | | attrs: tx_hash, tx_type, txq_status, | | + * | | fee_level_paid, required_fee_level | | + * | | | | + * | | +-------------------+ +----------------------+ | | + * | | | txq.apply_direct | | txq.batch_clear | | | + * | | | tryDirectApply() | | tryClearAccount...() | | | + * | | +-------------------+ | attrs: num_cleared | | | + * | | +----------------------+ | | + * | +--------------------------------------------------+ | + * +-------------------------------------------------------+ + * + * Ledger close (consensus thread): + * + * +-------------------------------------------------------+ + * | txq.accept | + * | TxQ::accept() | + * | attrs: queue_size, ledger_changed | + * | | + * | +--------------------------------------------------+ | + * | | txq.accept_tx (per queued transaction) | | + * | | attrs: tx_hash, ter_code, retries_remaining | | + * | +--------------------------------------------------+ | + * +-------------------------------------------------------+ + * + * Post-close cleanup: + * + * +-------------------------------------------------------+ + * | txq.cleanup | + * | TxQ::processClosedLedger() | + * | attrs: ledger_seq, expired_count | + * +-------------------------------------------------------+ + */ + +#include + +namespace xrpl::telemetry::txq_span { + +// ===== Span prefixes ======================================================= + +namespace prefix { +/** + * "txq" — root prefix for transaction queue spans. + */ +inline constexpr auto txq = makeStr("txq"); +} // namespace prefix + +// ===== Span operation suffixes ============================================= + +namespace op { +inline constexpr auto enqueue = makeStr("enqueue"); +inline constexpr auto applyDirect = makeStr("apply_direct"); +inline constexpr auto batchClear = makeStr("batch_clear"); +inline constexpr auto accept = makeStr("accept"); +inline constexpr auto acceptTx = makeStr("accept_tx"); +inline constexpr auto cleanup = makeStr("cleanup"); +} // namespace op + +// ===== Full span names (prefix.op) =========================================== +// +// Joined "txq." names for the explicit-context factories +// (SpanGuard::childSpan(name, ctx)) that take one full span name rather than a +// prefix/suffix pair. + +inline constexpr auto enqueue = join(prefix::txq, op::enqueue); + +// ===== Attribute keys ====================================================== + +namespace attr { +/** + * Canonical shared constants (defined in SpanNames.h). + */ +using ::xrpl::telemetry::attr::currentLedgerHash; +using ::xrpl::telemetry::attr::currentLedgerSeq; +using ::xrpl::telemetry::attr::ledgerSeq; +using ::xrpl::telemetry::attr::txHash; + +/** + * "txq_status" — domain-qualified (collides with tx_status, rpc_status). + */ +inline constexpr auto txqStatus = makeStr("txq_status"); +/** + * "fee_level_paid" — fee level paid by queued tx. + */ +inline constexpr auto feeLevelPaid = makeStr("fee_level_paid"); +/** + * "required_fee_level" — minimum fee level for inclusion. + */ +inline constexpr auto requiredFeeLevel = makeStr("required_fee_level"); +/** + * "queue_size" — current TxQ depth. + */ +inline constexpr auto queueSize = makeStr("queue_size"); +/** + * "ledger_changed" — whether ledger changed since last attempt. + */ +inline constexpr auto ledgerChanged = makeStr("ledger_changed"); +/** + * "expired_count" — number of expired entries cleared. + */ +inline constexpr auto expiredCount = makeStr("expired_count"); +/** + * "ter_code" — transaction engine result code. + */ +inline constexpr auto terCode = makeStr("ter_code"); +/** + * "retries_remaining" — retries left before discard. + */ +inline constexpr auto retriesRemaining = makeStr("retries_remaining"); +/** + * "num_cleared" — entries cleared in batch. + */ +inline constexpr auto numCleared = makeStr("num_cleared"); +/** + * "tx_type" — transaction type name (e.g., "Payment", "OfferCreate"). + */ +inline constexpr auto txType = makeStr("tx_type"); +} // namespace attr + +// ===== Attribute values ==================================================== + +namespace val { +inline constexpr auto queued = makeStr("queued"); +inline constexpr auto appliedDirect = makeStr("applied_direct"); +inline constexpr auto rejected = makeStr("rejected"); +inline constexpr auto applied = makeStr("applied"); +inline constexpr auto failed = makeStr("failed"); +inline constexpr auto retried = makeStr("retried"); +} // namespace val + +} // namespace xrpl::telemetry::txq_span diff --git a/src/xrpld/consensus/Consensus.h b/src/xrpld/consensus/Consensus.h index 440191939b8..4e890aca915 100644 --- a/src/xrpld/consensus/Consensus.h +++ b/src/xrpld/consensus/Consensus.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include @@ -24,6 +26,7 @@ #include #include #include +#include #include namespace xrpl { @@ -447,13 +450,29 @@ class Consensus getJson(bool full) const; private: + /** + * Why startRoundInternal is being entered. + * + * Distinguishes the normal Initial entry (from public startRound) + * from a Recovered re-entry (from handleWrongLedger after the + * correct prior ledger was acquired mid-round). The Recovered path + * resets phase to Open within the SAME round, which is a state + * transition that would otherwise be invisible in traces — the + * recovery flag drives a `phase.recovery` event on the round span. + */ + enum class StartRoundReason : std::uint8_t { + Initial, + Recovered, + }; + void startRoundInternal( NetClock::time_point const& now, Ledger_t::ID const& prevLedgerID, Ledger_t const& prevLedger, ConsensusMode mode, - std::unique_ptr const& clog); + std::unique_ptr const& clog, + StartRoundReason reason = StartRoundReason::Initial); // Change our view of the previous ledger void @@ -627,6 +646,47 @@ class Consensus // nodes that have bowed out of this consensus process hash_set deadNodes_; + /** + * Span for the establish phase of consensus. + * Created when the ledger closes and we enter phaseEstablish; + * cleared (ended) when consensus is reached. A thread-free SpanGuard, + * emplaced and reset() on different job workers. + */ + std::optional establishSpan_; + + /** + * Captured context of establishSpan_ (its own span context). Children + * (update_positions, check) build from this explicit context. + */ + xrpl::telemetry::SpanContext establishSpanContext_; + + /** + * Span for the open phase of consensus. + * Created in startRoundInternal(); cleared (ended) in closeLedger(). + * A thread-free SpanGuard, emplaced and reset() on different job workers. + */ + std::optional openSpan_; + + /** + * Create the establish-phase span if not yet active. + * Called on each phaseEstablish() invocation; no-op while span is live. + */ + void + startEstablishTracing(); + + /** + * Overwrite convergence metrics on the establish span each iteration. + * Final span attributes always reflect the last state before consensus. + */ + void + updateEstablishTracing(); + + /** + * End the establish span when transitioning to the accepted phase. + */ + void + endEstablishTracing(); + // Journal for debugging beast::Journal const j_; }; @@ -690,13 +750,50 @@ Consensus::startRoundInternal( Ledger_t::ID const& prevLedgerID, Ledger_t const& prevLedger, ConsensusMode mode, - std::unique_ptr const& clog) + std::unique_ptr const& clog, + StartRoundReason const reason) { + // Recovery path: handleWrongLedger acquired the correct prior ledger + // and re-entered startRoundInternal mid-round. The roundSpan_ owned by + // the adaptor is still the SAME span as before — startRoundTracing is + // not called on the recovery path, so we record the recovery as an + // event on the surviving round span. Pass empty phaseLabel to leave + // consensus_phase unchanged (the actual phase reset to open is marked + // separately by the new openSpan_ being emplaced below). + if (reason == StartRoundReason::Recovered) + { + adaptor_.onPhaseEvent(telemetry::consensus::span::event::phaseRecovery, ""); + } + phase_ = ConsensusPhase::Open; JLOG(j_.debug()) << "transitioned to ConsensusPhase::Open "; CLOG(clog) << "startRoundInternal transitioned to ConsensusPhase::Open, " "previous ledgerID: " << prevLedgerID << ", seq: " << prevLedger.seq() << ". "; + // Reset establishSpan_ so a wrongLedger recovery mid-establish doesn't + // leak the prior round's span into the new one (startEstablishTracing + // early-returns when establishSpan_ is populated). + establishSpan_.reset(); + establishSpanContext_ = telemetry::SpanContext{}; + // Child of the round span via its captured context: parent phase.open + // explicitly under roundSpanContext_. An invalid round context (round span + // not yet created) yields a null guard. openSpan_ is a thread-free + // SpanGuard emplaced here on one job worker and reset() on another; no + // scope to strip. + openSpan_.emplace( + telemetry::SpanGuard::childSpan( + telemetry::consensus::span::phaseOpen, adaptor_.roundSpanContext())); + // On the Recovered path, fire phase.open here because startRoundTracing + // (which fires it for the Initial path) is not called on re-entry. On + // the Initial path this is a no-op because the round span hasn't been + // created yet — the phase.open event is fired later by startRoundTracing + // after the new round span is in place. + if (reason == StartRoundReason::Recovered) + { + adaptor_.onPhaseEvent( + telemetry::consensus::span::event::phaseOpen, + telemetry::consensus::span::val::phaseOpen); + } mode_.set(mode, adaptor_); now_ = now; prevLedgerID_ = prevLedgerID; @@ -947,6 +1044,9 @@ Consensus::simulate( result_->proposers = prevProposers_ = currPeerPositions_.size(); prevRoundTime_ = result_->roundTime.read(); phase_ = ConsensusPhase::Accepted; + adaptor_.onPhaseEvent( + telemetry::consensus::span::event::phaseAccepted, + telemetry::consensus::span::val::phaseAccepted); adaptor_.onForceAccept( *result_, previousLedger_, closeResolution_, rawCloseTimes_, mode_.get(), getJson(true)); // NOLINTEND(bugprone-unchecked-optional-access) @@ -1095,7 +1195,13 @@ Consensus::handleWrongLedger( { JLOG(j_.info()) << "Have the consensus ledger " << prevLedgerID_; CLOG(clog) << "Have the consensus ledger " << prevLedgerID_ << ". "; - startRoundInternal(now_, lgrId, *newLedger, ConsensusMode::SwitchedLedger, clog); + startRoundInternal( + now_, + lgrId, + *newLedger, + ConsensusMode::SwitchedLedger, + clog, + StartRoundReason::Recovered); } else { @@ -1356,6 +1462,8 @@ Consensus::phaseEstablish(std::unique_ptr const& clo XRPL_ASSERT(result_, "xrpl::Consensus::phaseEstablish : result is set"); // NOLINTBEGIN(bugprone-unchecked-optional-access) assert above + startEstablishTracing(); + ++peerUnchangedCounter_; ++establishCounter_; @@ -1383,6 +1491,8 @@ Consensus::phaseEstablish(std::unique_ptr const& clo updateOurPositions(clog); + updateEstablishTracing(); + // Nothing to do if too many laggards or we don't have consensus. if (shouldPause(clog) || !haveConsensus(clog)) return; @@ -1400,7 +1510,26 @@ Consensus::phaseEstablish(std::unique_ptr const& clo adaptor_.updateOperatingMode(currPeerPositions_.size()); prevProposers_ = currPeerPositions_.size(); prevRoundTime_ = result_->roundTime.read(); + endEstablishTracing(); + { + namespace cs = telemetry::consensus::span; + if (result_->state == ConsensusState::Yes) + { + adaptor_.onOutcomeEvent(cs::event::outcomeYes); + } + else if (result_->state == ConsensusState::MovedOn) + { + adaptor_.onOutcomeEvent(cs::event::outcomeMovedOn); + } + else if (result_->state == ConsensusState::Expired) + { + adaptor_.onOutcomeEvent(cs::event::outcomeExpired); + } + } phase_ = ConsensusPhase::Accepted; + adaptor_.onPhaseEvent( + telemetry::consensus::span::event::phaseAccepted, + telemetry::consensus::span::val::phaseAccepted); JLOG(j_.debug()) << "transitioned to ConsensusPhase::Accepted"; adaptor_.onAccept( *result_, @@ -1420,7 +1549,23 @@ Consensus::closeLedger(std::unique_ptr const& clog) // We should not be closing if we already have a position XRPL_ASSERT(!result_, "xrpl::Consensus::closeLedger : result is not set"); + // Annotate the open-phase span with end-of-phase metadata before + // ending it, so a Tempo/Jaeger query for consensus.phase.open shows + // how long the phase ran and how many peer positions arrived during it. + openTime_.tick(clock_.now()); + if (openSpan_ && *openSpan_) + { + namespace cs = telemetry::consensus::span; + openSpan_->setAttribute( + cs::attr::openDurationMs, static_cast(openTime_.read().count())); + openSpan_->setAttribute( + cs::attr::peerPositionsAtClose, static_cast(currPeerPositions_.size())); + } + openSpan_.reset(); phase_ = ConsensusPhase::Establish; + adaptor_.onPhaseEvent( + telemetry::consensus::span::event::phaseEstablish, + telemetry::consensus::span::val::phaseEstablish); JLOG(j_.debug()) << "transitioned to ConsensusPhase::Establish"; rawCloseTimes_.self = now_; peerUnchangedCounter_ = 0; @@ -1477,6 +1622,17 @@ Consensus::updateOurPositions(std::unique_ptr const& // We must have a position if we are updating it XRPL_ASSERT(result_, "xrpl::Consensus::updateOurPositions : result is set"); // NOLINTBEGIN(bugprone-unchecked-optional-access) assert above + using namespace telemetry; + // Child of the establish span via its captured context (establishSpan_ is + // a thread-free SpanGuard, so parent explicitly via its context). Null + // context (establish not started) yields a null guard, same as before. + auto span = SpanGuard::childSpan(consensus::span::updatePositions, establishSpanContext_); + span.setAttribute( + consensus::span::attr::convergePercent, static_cast(convergePercent_)); + span.setAttribute( + consensus::span::attr::proposers, static_cast(currPeerPositions_.size())); + span.setAttribute( + consensus::span::attr::disputesCount, static_cast(result_->disputes.size())); ConsensusParms const& parms = adaptor_.parms(); // Compute a cutoff time @@ -1536,6 +1692,17 @@ Consensus::updateOurPositions(std::unique_ptr const& // now a no mutableSet->erase(txId); } + + auto const yaysStr = std::to_string(dispute.getYays()); + auto const naysStr = std::to_string(dispute.getNays()); + span.addEvent( + consensus::span::event::disputeResolve, + {{consensus::span::attr::txId, to_string(txId)}, + {consensus::span::attr::disputeOurVote, + dispute.getOurVote() ? std::string_view{consensus::span::val::yes} + : std::string_view{consensus::span::val::no}}, + {consensus::span::attr::disputeYays, yaysStr}, + {consensus::span::attr::disputeNays, naysStr}}); } } @@ -1560,6 +1727,8 @@ Consensus::updateOurPositions(std::unique_ptr const& if (newState) closeTimeAvalancheState_ = *newState; CLOG(clog) << "neededWeight " << neededWeight << ". "; + span.setAttribute( + consensus::span::attr::avalancheThreshold, static_cast(neededWeight)); int participants = currPeerPositions_.size(); if (mode_.get() == ConsensusMode::Proposing) @@ -1608,6 +1777,10 @@ Consensus::updateOurPositions(std::unique_ptr const& } } + span.setAttribute(consensus::span::attr::haveCloseTimeConsensus, haveCloseTimeConsensus_); + span.setAttribute( + consensus::span::attr::closeTimeThreshold, static_cast(parms.avCtConsensusPct)); + if (!ourNewSet && ((consensusCloseTime != asCloseTime(result_->position.closeTime())) || result_->position.isStale(ourCutoff))) @@ -1659,6 +1832,10 @@ Consensus::haveConsensus(std::unique_ptr const& clog // Must have a stance if we are checking for consensus XRPL_ASSERT(result_, "xrpl::Consensus::haveConsensus : has result"); // NOLINTBEGIN(bugprone-unchecked-optional-access) assert above + using namespace telemetry; + // Child of the establish span via its captured context (establishSpan_ is + // a thread-free SpanGuard, so parent explicitly via its context). + auto span = SpanGuard::childSpan(consensus::span::check, establishSpanContext_); // CHECKME: should possibly count unacquired TX sets as disagreeing int agree = 0, disagree = 0; @@ -1717,6 +1894,38 @@ Consensus::haveConsensus(std::unique_ptr const& clog j_, clog); + // Set span attributes before the early-return branches below so the + // consensus.check span carries diagnostic data even when consensus is + // not reached (the No / Expired paths return early). + span.setAttribute(consensus::span::attr::agreeCount, static_cast(agree)); + span.setAttribute(consensus::span::attr::disagreeCount, static_cast(disagree)); + span.setAttribute( + consensus::span::attr::convergePercent, static_cast(convergePercent_)); + span.setAttribute(consensus::span::attr::haveCloseTimeConsensus, haveCloseTimeConsensus_); + span.setAttribute( + consensus::span::attr::thresholdPercent, + static_cast(adaptor_.parms().avCtConsensusPct)); + span.setAttribute( + consensus::span::attr::proposersFinished, static_cast(currentFinished)); + span.setAttribute(consensus::span::attr::consensusStalled, stalled); + span.setAttribute( + consensus::span::attr::establishCount, static_cast(establishCounter_)); + + std::string_view stateStr = consensus::span::val::no; + if (result_->state == ConsensusState::Yes) + { + stateStr = consensus::span::val::yes; + } + else if (result_->state == ConsensusState::MovedOn) + { + stateStr = consensus::span::val::movedOn; + } + else if (result_->state == ConsensusState::Expired) + { + stateStr = consensus::span::val::expired; + } + span.setAttribute(consensus::span::attr::consensusResult, stateStr); + if (result_->state == ConsensusState::No) { CLOG(clog) << "No consensus. "; @@ -1879,4 +2088,52 @@ Consensus::asCloseTime(NetClock::time_point raw) const return roundCloseTime(raw, closeResolution_); } +template +void +Consensus::startEstablishTracing() +{ + if (establishSpan_) + return; + // Child of the round span via its captured context: parent establish + // explicitly under roundSpanContext_. An invalid round context (round span + // not yet created) yields a null guard. + establishSpan_.emplace( + telemetry::SpanGuard::childSpan( + telemetry::consensus::span::establish, adaptor_.roundSpanContext())); + // Capture the establish context; children (update_positions, check) parent + // to it explicitly via establishSpanContext_. establishSpan_ is a + // thread-free SpanGuard reset() on a different worker than it is emplaced + // on -- no scope work. + if (*establishSpan_) + { + establishSpanContext_ = establishSpan_->spanContext(); + } +} + +template +void +Consensus::updateEstablishTracing() +{ + if (!establishSpan_) + return; + namespace cs = telemetry::consensus::span; + establishSpan_->setAttribute(cs::attr::convergePercent, static_cast(convergePercent_)); + establishSpan_->setAttribute(cs::attr::establishCount, static_cast(establishCounter_)); + establishSpan_->setAttribute( + cs::attr::proposers, static_cast(currPeerPositions_.size())); + if (result_) + { + establishSpan_->setAttribute( + cs::attr::disputesCount, static_cast(result_->disputes.size())); + } +} + +template +void +Consensus::endEstablishTracing() +{ + establishSpan_.reset(); + establishSpanContext_ = telemetry::SpanContext{}; +} + } // namespace xrpl diff --git a/src/xrpld/consensus/ConsensusSpanNames.h b/src/xrpld/consensus/ConsensusSpanNames.h new file mode 100644 index 00000000000..6c79cd3132a --- /dev/null +++ b/src/xrpld/consensus/ConsensusSpanNames.h @@ -0,0 +1,296 @@ +#pragma once + +/** + * Compile-time span name constants for consensus tracing. + * + * Used by RCLConsensus (app), Consensus.h (template), and PeerImp + * (overlay) for consensus lifecycle spans. + * Built on StaticStr/join() from SpanNames.h. + * + * ## Span Hierarchy + * + * Root span created in Adaptor::startRoundTracing(). In "deterministic" + * strategy the trace-id is derived from the previous ledger hash so all + * nodes tracing the same round share a trace. + * + * consensus.round [main thread, root] + * | Created: Adaptor::startRoundTracing() + * | Attrs: consensus_ledger_id, ledger_seq, consensus_mode, + * | trace_strategy, consensus_round_id + * | + * +-- consensus.phase.open [main thread, child] + * | Created: Consensus::startRoundInternal() + * | Ended: Consensus::closeLedger() + * | + * +-- consensus.proposal.send [main thread] + * | Created: Adaptor::propose() + * | Attrs: consensus_round (proposeSeq) + * | + * +-- consensus.ledger_close [main thread] + * | Created: Adaptor::onClose() + * | Attrs: ledger_seq, consensus_mode + * | + * +-- consensus.establish [main thread, child] + * | Created: Consensus::startEstablishTracing() + * | Ended: Consensus::phaseEstablish() on accept + * | Attrs: converge_percent, establish_count, proposers + * | + * +-- consensus.update_positions [main thread] + * | Created: Consensus::updateOurPositions() + * | Attrs: converge_percent, proposers, disputes_count + * | Events: per-dispute vote details (tx_id, our_vote, yays, nays) + * | + * +-- consensus.check [main thread] + * | Created: Consensus::haveConsensus() + * | Attrs: agree/disagree counts, threshold_percent, result + * | + * +-- consensus.accept [main thread, child of round] + * | Created: Adaptor::makeAcceptSpan(), shared_ptr kept alive + * | until doAccept() completes on jtACCEPT thread + * | Attrs: proposers, round_time_ms, quorum + * | | + * | +-- consensus.accept.apply [jtACCEPT thread, child of accept] + * | Created: Adaptor::doAccept() + * | Attrs: ledger_seq, close_time, close_time_correct, + * | close_resolution_ms, consensus_state, proposing, round_time_ms, + * | parent_close_time, close_time_self, close_time_vote_bins, + * | resolution_direction, tx_count + * | Events: tx.included (per tx, attrs: tx_id) + * | + * +~~~ consensus.validation.send [jtACCEPT thread, linked] + * | Created: Adaptor::createValidationSpan() (follows-from link) + * | Attrs: ledger_seq, proposing + * | + * +-- consensus.mode_change [main thread] + * Created: Adaptor::onModeChange() + * Attrs: mode_old, mode_new + * + * Standalone spans (no parent, created per-message in overlay): + * + * consensus.proposal.receive [PeerImp I/O thread] + * Created: PeerImp::onMessage(TMProposeSet) + * + * consensus.validation.receive [PeerImp I/O thread] + * Created: PeerImp::onMessage(TMValidation) + * + * Legend: + * +-- child-of relationship (same trace) + * +~~~ follows-from link (separate sub-tree, causal link) + */ + +#include + +namespace xrpl::telemetry::consensus::span { + +// ===== Span name segments ==================================================== + +namespace part { +inline constexpr auto proposal = makeStr("proposal"); +inline constexpr auto validation = makeStr("validation"); +inline constexpr auto accept = makeStr("accept"); +inline constexpr auto phase = makeStr("phase"); +} // namespace part + +namespace op { +inline constexpr auto round = makeStr("round"); +inline constexpr auto proposalSend = join(part::proposal, makeStr("send")); +inline constexpr auto ledgerClose = makeStr("ledger_close"); +inline constexpr auto establish = makeStr("establish"); +inline constexpr auto updatePositions = makeStr("update_positions"); +inline constexpr auto check = makeStr("check"); +inline constexpr auto accept = makeStr("accept"); +inline constexpr auto acceptApply = join(part::accept, makeStr("apply")); +inline constexpr auto validationSend = join(part::validation, makeStr("send")); +inline constexpr auto modeChange = makeStr("mode_change"); +inline constexpr auto proposalReceive = join(part::proposal, makeStr("receive")); +inline constexpr auto validationReceive = join(part::validation, makeStr("receive")); +inline constexpr auto phaseOpen = join(part::phase, makeStr("open")); +} // namespace op + +// ===== Full span names (prefix.op) =========================================== + +inline constexpr auto round = join(seg::consensus, op::round); +inline constexpr auto proposalSend = join(seg::consensus, op::proposalSend); +inline constexpr auto ledgerClose = join(seg::consensus, op::ledgerClose); +inline constexpr auto establish = join(seg::consensus, op::establish); +inline constexpr auto updatePositions = join(seg::consensus, op::updatePositions); +inline constexpr auto check = join(seg::consensus, op::check); +inline constexpr auto accept = join(seg::consensus, op::accept); +inline constexpr auto acceptApply = join(seg::consensus, op::acceptApply); +inline constexpr auto validationSend = join(seg::consensus, op::validationSend); +inline constexpr auto modeChange = join(seg::consensus, op::modeChange); +inline constexpr auto proposalReceive = join(seg::consensus, op::proposalReceive); +inline constexpr auto validationReceive = join(seg::consensus, op::validationReceive); +inline constexpr auto phaseOpen = join(seg::consensus, op::phaseOpen); + +// ===== Attribute keys ======================================================== + +namespace attr { +/** + * Canonical shared constants (defined in SpanNames.h). `ledgerHash` and + * `fullValidation` are shared with the peer.validation.receive span — same + * concept, same key, distinguished by span name (not an emitter prefix). + */ +using ::xrpl::telemetry::attr::closeResolutionMs; +using ::xrpl::telemetry::attr::closeTime; +using ::xrpl::telemetry::attr::closeTimeCorrect; +using ::xrpl::telemetry::attr::fullValidation; +using ::xrpl::telemetry::attr::ledgerHash; +using ::xrpl::telemetry::attr::ledgerSeq; + +/** + * Domain-qualified attrs (rule 5 — bare name ambiguous across domains). + * Use `_` underscore form for TraceQL ergonomics. + */ +inline constexpr auto ledgerId = makeStr("consensus_ledger_id"); +inline constexpr auto mode = makeStr("consensus_mode"); +inline constexpr auto round = makeStr("consensus_round"); +inline constexpr auto roundId = makeStr("consensus_round_id"); +/** + * Current phase name attached to consensus.round; updated on each + * phase transition event (open/establish/accepted). + */ +inline constexpr auto consensusPhase = makeStr("consensus_phase"); +/** + * Boolean flag set on consensus.check when checkConsensus reports stalled. + */ +inline constexpr auto consensusStalled = makeStr("consensus_stalled"); + +/** + * Domain-owned bare attrs. + */ +inline constexpr auto proposers = makeStr("proposers"); +inline constexpr auto roundTimeMs = makeStr("round_time_ms"); +inline constexpr auto proposing = makeStr("proposing"); +/** + * Round continuity / context attrs (set on consensus.round at round start). + */ +inline constexpr auto previousProposers = makeStr("previous_proposers"); +inline constexpr auto previousRoundTimeMs = makeStr("previous_round_time_ms"); +inline constexpr auto previousLedgerSeq = makeStr("previous_ledger_seq"); +inline constexpr auto closeTimeResolutionMs = makeStr("close_time_resolution_ms"); +/** + * Open-phase end metadata (set on consensus.phase.open before reset). + */ +inline constexpr auto openDurationMs = makeStr("open_duration_ms"); +inline constexpr auto peerPositionsAtClose = makeStr("peer_positions_at_close"); +/** + * Ledger-close inputs. + */ +inline constexpr auto txCountOpen = makeStr("tx_count_open"); +/** + * Establish/check additional state. + */ +inline constexpr auto proposersFinished = makeStr("proposers_finished"); +/** + * Accept/apply enrichment. + */ +inline constexpr auto disputesResolvedCount = makeStr("disputes_resolved_count"); +/** + * Validation send/receive enrichment. (`full_validation` is shared — see the + * `using` re-export above.) + */ +inline constexpr auto validationSignTime = makeStr("validation_sign_time"); +/** + * Receive-side hash prefixes for cross-peer correlation. + */ +inline constexpr auto prevLedgerPrefix = makeStr("prev_ledger_prefix"); +inline constexpr auto positionHashPrefix = makeStr("position_hash_prefix"); +/** + * "consensus_state" — domain-qualified (collides with other domains' state). + */ +inline constexpr auto consensusState = makeStr("consensus_state"); +inline constexpr auto parentCloseTime = makeStr("parent_close_time"); +inline constexpr auto closeTimeSelf = makeStr("close_time_self"); +inline constexpr auto closeTimeVoteBins = makeStr("close_time_vote_bins"); +inline constexpr auto resolutionDirection = makeStr("resolution_direction"); +inline constexpr auto convergePercent = makeStr("converge_percent"); +inline constexpr auto establishCount = makeStr("establish_count"); +inline constexpr auto avalancheThreshold = makeStr("avalanche_threshold"); +inline constexpr auto closeTimeThreshold = makeStr("close_time_threshold"); +inline constexpr auto haveCloseTimeConsensus = makeStr("have_close_time_consensus"); +inline constexpr auto agreeCount = makeStr("agree_count"); +inline constexpr auto disagreeCount = makeStr("disagree_count"); +inline constexpr auto thresholdPercent = makeStr("threshold_percent"); +/** + * "consensus_result" — domain-qualified (collides with generic result). + */ +inline constexpr auto consensusResult = makeStr("consensus_result"); +inline constexpr auto quorum = makeStr("quorum"); +inline constexpr auto traceStrategy = makeStr("trace_strategy"); +inline constexpr auto modeOld = makeStr("mode_old"); +inline constexpr auto modeNew = makeStr("mode_new"); + +/** + * "is_bow_out" — whether this proposal is a bow-out (resigning from round). + */ +inline constexpr auto isBowOut = makeStr("is_bow_out"); + +/** + * Transaction/dispute attrs used in consensus accept spans. + */ +inline constexpr auto txId = makeStr("tx_id"); +inline constexpr auto disputeOurVote = makeStr("dispute_our_vote"); +inline constexpr auto disputeYays = makeStr("dispute_yays"); +inline constexpr auto disputeNays = makeStr("dispute_nays"); +inline constexpr auto txCount = makeStr("tx_count"); +inline constexpr auto disputesCount = makeStr("disputes_count"); +/** + * Trust flag (is the message origin a trusted UNL validator). Qualified by + * message type, shared with the peer.{proposal,validation}.receive spans: + * consensus.proposal.receive uses `proposal_trusted`, consensus.validation. + * receive uses `validation_trusted`. Same concept on both emitters → same key. + */ +inline constexpr auto proposalTrusted = makeStr("proposal_trusted"); +inline constexpr auto validationTrusted = makeStr("validation_trusted"); +} // namespace attr + +// ===== Event names =========================================================== + +namespace event { +/** + * "dispute.resolve" + */ +inline constexpr auto disputeResolve = join(makeStr("dispute"), makeStr("resolve")); +/** + * "tx.included" + */ +inline constexpr auto txIncluded = join(makeStr("tx"), makeStr("included")); + +/** + * Phase transition events — fired on consensus.round at each transition + * so the round-level span carries a complete timeline of phase changes, + * including the handleWrongLedger recovery edge that re-enters Open. + */ +inline constexpr auto phaseOpen = join(makeStr("phase"), makeStr("open")); +inline constexpr auto phaseEstablish = join(makeStr("phase"), makeStr("establish")); +inline constexpr auto phaseAccepted = join(makeStr("phase"), makeStr("accepted")); +inline constexpr auto phaseRecovery = join(makeStr("phase"), makeStr("recovery")); + +/** + * Outcome events — fired on consensus.round at the establish→accepted + * transition so the path that drove acceptance is queryable. + */ +inline constexpr auto outcomeYes = join(makeStr("outcome"), makeStr("yes")); +inline constexpr auto outcomeMovedOn = join(makeStr("outcome"), makeStr("moved_on")); +inline constexpr auto outcomeExpired = join(makeStr("outcome"), makeStr("expired")); +} // namespace event + +// ===== Attribute values ====================================================== + +namespace val { +inline constexpr auto finished = makeStr("finished"); +inline constexpr auto movedOn = makeStr("moved_on"); +inline constexpr auto yes = makeStr("yes"); +inline constexpr auto no = makeStr("no"); +inline constexpr auto expired = makeStr("expired"); +inline constexpr auto increased = makeStr("increased"); +inline constexpr auto decreased = makeStr("decreased"); +inline constexpr auto unchanged = makeStr("unchanged"); +// consensus_phase attribute values (the phase the round is entering). +inline constexpr auto phaseOpen = makeStr("open"); +inline constexpr auto phaseEstablish = makeStr("establish"); +inline constexpr auto phaseAccepted = makeStr("accepted"); +} // namespace val + +} // namespace xrpl::telemetry::consensus::span diff --git a/src/xrpld/consensus/ConsensusTypes.h b/src/xrpld/consensus/ConsensusTypes.h index 4553e46f482..69c1a55aac2 100644 --- a/src/xrpld/consensus/ConsensusTypes.h +++ b/src/xrpld/consensus/ConsensusTypes.h @@ -79,10 +79,32 @@ to_string(ConsensusMode m) } } +/** + * Title Case display name for telemetry attributes and dashboards. + * Separate from to_string() which is used in logs and must remain stable. + */ +inline std::string +toDisplayString(ConsensusMode m) +{ + switch (m) + { + case ConsensusMode::Proposing: + return "Proposing"; + case ConsensusMode::Observing: + return "Observing"; + case ConsensusMode::WrongLedger: + return "Wrong Ledger"; + case ConsensusMode::SwitchedLedger: + return "Switched Ledger"; + default: + return "Unknown"; + } +} + /** * Phases of consensus for a single ledger round. * - * @code + * @code * "close" "accept" * open ------- > establish ---------> accepted * ^ | | diff --git a/src/xrpld/consensus/DisputedTx.h b/src/xrpld/consensus/DisputedTx.h index 12ed00d4607..6fc2912c831 100644 --- a/src/xrpld/consensus/DisputedTx.h +++ b/src/xrpld/consensus/DisputedTx.h @@ -198,6 +198,24 @@ class DisputedTx [[nodiscard]] json::Value getJson() const; + /** + * Number of peers voting yes. + */ + [[nodiscard]] int + getYays() const + { + return yays_; + } + + /** + * Number of peers voting no. + */ + [[nodiscard]] int + getNays() const + { + return nays_; + } + private: int yays_{0}; //< Number of yes votes int nays_{0}; //< Number of no votes diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 8838970b5ff..aebb7c281b7 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -1,3 +1,7 @@ +// cspell:ignore ISTOGRAM +// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's +// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. + #include #include @@ -8,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -15,12 +20,17 @@ #include #include #include +#include #include #include #include #include #include #include +#include +#include +#include +#include #include #include @@ -51,6 +61,7 @@ #include #include #include +#include #include #include #include @@ -62,6 +73,9 @@ #include #include #include +#include +#include +#include #include #include @@ -1306,6 +1320,25 @@ PeerImp::handleTransaction( auto stx = std::make_shared(sit); uint256 const txID = stx->getTransactionID(); + using namespace telemetry; + // SpanGuard is thread-free (holds no Scope), so it is safe to hand to + // a job-queue worker and end on that thread — no detach step is needed. + auto span = std::make_shared(txReceiveSpan(txID, *m)); + span->setAttribute(tx_span::attr::txHash, to_string(txID).c_str()); + span->setAttribute(tx_span::attr::peerId, static_cast(id_)); + // The current (open) ledger index when the relayed tx was received — + // the ledger being worked on. Correlates this tx.receive to the ledger + // trace; not yet applied to a specific ledger here, so no hash. + span->setAttribute( + tx_span::attr::currentLedgerSeq, + static_cast(app_.getLedgerMaster().getCurrentLedgerIndex())); + if (auto const* fmt = TxFormats::getInstance().findByType(stx->getTxnType())) + span->setAttribute(tx_span::attr::txType, fmt->getName().c_str()); + if (auto const version = getVersion(); !version.empty()) + span->setAttribute(tx_span::attr::peerVersion, version.c_str()); + // Note: suppressed and txStatus are set once at each exit path + // (not as defaults here) to avoid OTel SDK attribute duplication. + // Charge strongly for attempting to relay a txn with tfInnerBatchTxn // LCOV_EXCL_START /* @@ -1326,6 +1359,7 @@ PeerImp::handleTransaction( */ if (stx->isFlag(tfInnerBatchTxn)) { + span->setAttribute(tx_span::attr::txStatus, tx_span::val::rejectedInnerBatch); JLOG(pJournal_.warn()) << "Ignoring Network relayed Tx containing " "tfInnerBatchTxn (handleTransaction)."; fee_.update(Resource::kFeeModerateBurdenPeer, "inner batch txn"); @@ -1338,18 +1372,28 @@ PeerImp::handleTransaction( if (!app_.getHashRouter().shouldProcess(txID, id_, flags, kTxInterval)) { + span->setAttribute(tx_span::attr::suppressed, true); // we have seen this transaction recently if (any(flags & HashRouterFlags::BAD)) { + span->setAttribute(tx_span::attr::txStatus, tx_span::val::knownBad); fee_.update(Resource::kFeeUselessData, "known bad"); JLOG(pJournal_.debug()) << "Ignoring known bad tx " << txID; } - - // Erase only if the server has seen this tx. If the server has not - // seen this tx then the tx could not has been queued for this peer. - else if (eraseTxQueue && txReduceRelayEnabled()) + else { - removeTxQueue(txID); + // Recently-seen but not flagged bad — this is the plain + // duplicate-suppression path. Mark it explicitly so the + // span never exits as "new". + span->setAttribute(tx_span::attr::txStatus, tx_span::val::suppressed); + + // Erase only if the server has seen this tx. If the server + // has not seen this tx then the tx could not have been + // queued for this peer. + if (eraseTxQueue && txReduceRelayEnabled()) + { + removeTxQueue(txID); + } } overlay_.reportInboundTraffic( @@ -1358,6 +1402,7 @@ PeerImp::handleTransaction( return; } + span->setAttribute(tx_span::attr::suppressed, false); JLOG(pJournal_.debug()) << "Got tx " << txID; bool checkSignature = true; @@ -1382,10 +1427,12 @@ PeerImp::handleTransaction( if (app_.getLedgerMaster().getValidatedLedgerAge() > 4min) { + span->setAttribute(tx_span::attr::txStatus, tx_span::val::droppedNoSync); JLOG(pJournal_.trace()) << "No new transactions until synchronized"; } else if (app_.getJobQueue().getJobCount(JtTransaction) > app_.config().maxTransactions) { + span->setAttribute(tx_span::attr::txStatus, tx_span::val::droppedQueueFull); overlay_.incJqTransOverflow(); JLOG(pJournal_.info()) << "Transaction queue is full"; } @@ -1398,7 +1445,12 @@ PeerImp::handleTransaction( flags, checkSignature, batch, - stx]() { + stx, + sp = std::move(span)]() { + // Activate the tx.receive span so checkTransaction's log + // lines carry its trace_id. Non-owning; sp still owns/ends + // the span. This job body runs to completion on one worker. + auto activation = telemetry::activateIfLive(sp); if (auto peer = weak.lock()) peer->checkTransaction(flags, checkSignature, stx, batch); }); @@ -1719,6 +1771,13 @@ PeerImp::onMessage(std::shared_ptr const& m) void PeerImp::onMessage(std::shared_ptr const& m) { + using namespace telemetry; + // root: inbound peer message entry point (kConsumer); must not inherit + // any span left active on this peer thread. + auto span = + ScopedSpanGuard::freshRoot(TraceCategory::Peer, seg::peer, peer_span::op::proposalReceive); + span.setAttribute(peer_span::attr::peerId, static_cast(id_)); + protocol::TMProposeSet const& set = *m; auto const sig = makeSlice(set.signature()); @@ -1745,6 +1804,7 @@ PeerImp::onMessage(std::shared_ptr const& m) // every time a spam packet is received PublicKey const publicKey{makeSlice(set.nodepubkey())}; auto const isTrusted = app_.getValidators().trusted(publicKey); + span.setAttribute(peer_span::attr::proposalTrusted, isTrusted); // If the operator has specified that untrusted proposals be dropped then // this happens here I.e. before further wasting CPU verifying the signature @@ -1813,9 +1873,29 @@ PeerImp::onMessage(std::shared_ptr const& m) app_.getTimeKeeper().closeTime(), calcNodeID(app_.getValidatorManifests().getMasterKey(publicKey))}); + // Create a receive span that links to the sender's trace context + // (if propagated). shared_ptr keeps it alive across the job boundary. + // The receive span is a thread-free SpanGuard handed to the job worker; + // no scope to strip. + auto consSpan = std::make_shared(telemetry::proposalReceiveSpan(set)); + consSpan->setAttribute(telemetry::consensus::span::attr::proposalTrusted, isTrusted); + consSpan->setAttribute( + telemetry::consensus::span::attr::round, static_cast(set.proposeseq())); + // First 16 hex chars (8 bytes) of each hash — enough to disambiguate + // peer positions and prior ledgers without exporting full 32-byte + // hashes on every receive event. + consSpan->setAttribute( + telemetry::consensus::span::attr::prevLedgerPrefix, + to_string(prevLedger).substr(0, 16).c_str()); + consSpan->setAttribute( + telemetry::consensus::span::attr::positionHashPrefix, + to_string(proposeHash).substr(0, 16).c_str()); + std::weak_ptr const weak = shared_from_this(); app_.getJobQueue().addJob( - isTrusted ? JtProposalT : JtProposalUt, "checkPropose", [weak, isTrusted, m, proposal]() { + isTrusted ? JtProposalT : JtProposalUt, + "checkPropose", + [weak, isTrusted, m, proposal, sp = std::move(consSpan)]() { if (auto peer = weak.lock()) peer->checkPropose(isTrusted, m, proposal); }); @@ -2315,6 +2395,13 @@ PeerImp::onMessage(std::shared_ptr const& m void PeerImp::onMessage(std::shared_ptr const& m) { + using namespace telemetry; + // root: inbound peer message entry point (kConsumer); must not inherit + // any span left active on this peer thread. + auto valSpan = ScopedSpanGuard::freshRoot( + TraceCategory::Peer, seg::peer, peer_span::op::validationReceive); + valSpan.setAttribute(peer_span::attr::peerId, static_cast(id_)); + if (m->validation().size() < 50) { JLOG(pJournal_.warn()) << "Validation: Too small"; @@ -2337,6 +2424,8 @@ PeerImp::onMessage(std::shared_ptr const& m) false); val->setSeen(closeTime); } + valSpan.setAttribute(peer_span::attr::ledgerHash, to_string(val->getLedgerHash()).c_str()); + valSpan.setAttribute(peer_span::attr::fullValidation, val->isFull()); if (!isCurrent( app_.getValidations().parms(), @@ -2353,6 +2442,7 @@ PeerImp::onMessage(std::shared_ptr const& m) // suppression for 30 seconds to avoid doing a relatively expensive // lookup every time a spam packet is received auto const isTrusted = app_.getValidators().trusted(val->getSignerPublic()); + valSpan.setAttribute(peer_span::attr::validationTrusted, isTrusted); // If the operator has specified that untrusted validations be // dropped then this happens here I.e. before further wasting CPU @@ -2390,6 +2480,24 @@ PeerImp::onMessage(std::shared_ptr const& m) return; } + // Create a receive span that links to the sender's trace context + // (if propagated). shared_ptr keeps it alive across the job boundary. + // The receive span is a thread-free SpanGuard handed to the job worker; + // no scope to strip. + auto consSpan = + std::make_shared(telemetry::validationReceiveSpan(*m)); + consSpan->setAttribute(telemetry::consensus::span::attr::validationTrusted, isTrusted); + if (val->isFieldPresent(sfLedgerSequence)) + { + consSpan->setAttribute( + telemetry::consensus::span::attr::ledgerSeq, + static_cast(val->getFieldU32(sfLedgerSequence))); + } + consSpan->setAttribute(telemetry::consensus::span::attr::fullValidation, val->isFull()); + consSpan->setAttribute( + telemetry::consensus::span::attr::validationSignTime, + static_cast(val->getSignTime().time_since_epoch().count())); + if (!isTrusted && (tracking_.load() == Tracking::Diverged)) { JLOG(pJournal_.debug()) << "Dropping untrusted validation from diverged peer"; @@ -2400,7 +2508,9 @@ PeerImp::onMessage(std::shared_ptr const& m) std::weak_ptr const weak = shared_from_this(); app_.getJobQueue().addJob( - isTrusted ? JtValidationT : JtValidationUt, name, [weak, val, m, key]() { + isTrusted ? JtValidationT : JtValidationUt, + name, + [weak, val, m, key, sp = std::move(consSpan)]() { if (auto peer = weak.lock()) peer->checkValidation(val, key, m); }); @@ -2464,6 +2574,12 @@ PeerImp::onMessage(std::shared_ptr const& m) { JLOG(pJournal_.debug()) << "GetObj: malformed ledgerhash from peer " << id_; fee_.update(Resource::kFeeMalformedRequest, "get object ledger hash"); + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::kGetObjectRejectedTotal, + telemetry::kGetObjectRejectedTotalDesc, + {{telemetry::kLabelReason, + std::string(telemetry::kReasonMalformedLedgerHash)}}); return; } } @@ -2476,6 +2592,11 @@ PeerImp::onMessage(std::shared_ptr const& m) << "GetObj: oversized request from peer " << id_ << " (" << packet.objects_size() << " > " << Tuning::kHardMaxReplyNodes << ")"; fee_.update(Resource::kFeeInvalidData, "oversized get object request"); + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::kGetObjectRejectedTotal, + telemetry::kGetObjectRejectedTotalDesc, + {{telemetry::kLabelReason, std::string(telemetry::kReasonOversize)}}); return; } @@ -2595,6 +2716,11 @@ PeerImp::processGetObjectByHash(std::shared_ptr con int const requested = packet.objects_size(); int const iterLimit = std::min(requested, Tuning::kHardMaxReplyNodes); + // Time the whole loop once, not each iteration: the loop can run up to + // kHardMaxReplyNodes times, so per-iteration clock reads would cost more + // than the lookups they measure. + auto const lookupStart = std::chrono::steady_clock::now(); + for (int i = 0; i < iterLimit; ++i) { auto const& obj = packet.objects(i); @@ -2619,20 +2745,74 @@ PeerImp::processGetObjectByHash(std::shared_ptr con newObj.set_ledgerseq(obj.ledgerseq()); } + auto const lookupElapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - lookupStart); + // Apply work-proportional charge. `charge()` posts the disconnect // step (if any) back to strand_, so it is safe to call from this // JobQueue worker thread. - charge( - // We pass `requested` directly here, instead of actual lookups done. Which could be - // std::min(packet.objects_size(), static_cast(Tuning::kHardMaxReplyNodes)); - // Because we want to charge as per the request size, to discourage large requests. - computeGetObjectByHashFee(requested, reply.objects_size()), - "processed get object by hash request"); + // + // We pass `requested` directly here, instead of actual lookups done. Which could be + // std::min(packet.objects_size(), static_cast(Tuning::kHardMaxReplyNodes)); + // Because we want to charge as per the request size, to discourage large requests. + // + // Computed into a local so the recorded metric is exactly the charge + // that is applied -- calling the helper twice could diverge. + Resource::Charge const fee = computeGetObjectByHashFee(requested, reply.objects_size()); + charge(fee, "processed get object by hash request"); + + recordGetObjectMetrics(requested, reply.objects_size(), lookupElapsed, fee); JLOG(pJournal_.trace()) << "GetObj: " << reply.objects_size() << " of " << requested; send(std::make_shared(reply, protocol::mtGET_OBJECTS)); } +void +PeerImp::recordGetObjectMetrics( + int const requested, + int const found, + std::chrono::microseconds const lookupElapsed, + Resource::Charge const& fee) +{ + using namespace telemetry; + + XRPL_METRIC_HISTOGRAM_RECORD( + app_, kGetObjectRequestObjects, kGetObjectRequestObjectsDesc, requested); + + XRPL_METRIC_HISTOGRAM_RECORD( + app_, kGetObjectLookupUs, kGetObjectLookupUsDesc, lookupElapsed.count()); + + XRPL_METRIC_HISTOGRAM_RECORD(app_, kGetObjectCharge, kGetObjectChargeDesc, fee.cost()); + + // Batch totals, added once per request rather than once per object: + // per-object increments on a loop bounded by kHardMaxReplyNodes would be + // a measurable cost for no extra information. + // + // `found` is the reply size, which the fetch loop only grows on a + // successful lookup within `iterLimit <= requested`, so `found <= + // requested` always holds. std::max still clamps both values, so a future + // caller passing found > requested cannot make the miss count wrap + // negative -- the counter takes an unsigned amount, where a wrap would + // read as ~1.8e19 rather than as an error. + // + // Written as two calls rather than a loop over a {hit, miss} pair: the + // macros expand to empty statements in a telemetry-off build, which would + // leave a loop's induction variable unused and fail the -Werror build. + XRPL_METRIC_COUNTER_ADD_LABELED( + app_, + kGetObjectLookupsTotal, + kGetObjectLookupsTotalDesc, + static_cast(std::max(0, found)), + {{kLabelResult, std::string(kResultHit)}}); + + XRPL_METRIC_COUNTER_ADD_LABELED( + app_, + kGetObjectLookupsTotal, + kGetObjectLookupsTotalDesc, + static_cast(std::max(0, requested - found)), + {{kLabelResult, std::string(kResultMiss)}}); +} + void PeerImp::onMessage(std::shared_ptr const& m) { diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index ea6eccd6563..2e004f0f3f0 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -681,6 +681,34 @@ class PeerImp : public Peer, public std::enable_shared_from_this, publi void processLedgerRequest(std::shared_ptr const& m); + /** + * Record the OTel metrics for one completed `TMGetObjectByHash` request. + * + * Extracted from `processGetObjectByHash()` purely to keep that method + * within the 80-line limit; it holds no logic of its own beyond deriving + * the hit/miss split from `requested` and `found`. Called once per + * request, after the fetch loop and the `charge()` call. + * + * Records `getobject_request_objects`, `getobject_lookup_us`, + * `getobject_charge`, and both label values of + * `getobject_lookups_total`. Compiles to nothing when telemetry is + * disabled, because the `XRPL_METRIC_*` macros do. + * + * @param requested Objects the peer asked for (`objects_size()`). + * @param found Objects returned, i.e. the reply's object count. + * Expected to be `<= requested`; clamped either way + * so the derived miss count cannot go negative. + * @param lookupElapsed Wall time of the whole fetch loop. + * @param fee The dynamic charge that was applied, so the + * recorded value is exactly the one charged. + */ + void + recordGetObjectMetrics( + int const requested, + int const found, + std::chrono::microseconds const lookupElapsed, + Resource::Charge const& fee); + protected: // Kept `protected` so test subclasses (see // TMGetObjectByHash_test) can drive the diff --git a/src/xrpld/overlay/detail/PeerSpanNames.h b/src/xrpld/overlay/detail/PeerSpanNames.h new file mode 100644 index 00000000000..6212d8fca25 --- /dev/null +++ b/src/xrpld/overlay/detail/PeerSpanNames.h @@ -0,0 +1,45 @@ +#pragma once + +/** + * Compile-time span name constants for peer overlay tracing. + * + * Used by PeerImp for peer message handling spans (proposals, + * validations). Built on StaticStr/join() from SpanNames.h. + * + * Span hierarchy: + * + * peer.proposal.receive (PeerImp — incoming proposal) + * peer.validation.receive (PeerImp — incoming validation) + */ + +#include + +namespace xrpl::telemetry::peer_span { + +// ===== Span operation suffixes =============================================== + +namespace op { +inline constexpr auto proposalReceive = makeStr("proposal.receive"); +inline constexpr auto validationReceive = makeStr("validation.receive"); +} // namespace op + +// ===== Attribute keys ======================================================== + +namespace attr { +/** + * Canonical shared constants (defined in SpanNames.h). `ledgerHash` and + * `fullValidation` are shared with the consensus validation spans — same + * concept, same key, told apart by span name. + */ +using ::xrpl::telemetry::attr::fullValidation; +using ::xrpl::telemetry::attr::ledgerHash; +using ::xrpl::telemetry::attr::peerId; + +/** + * Trust flag qualified by message type, shared with consensus.*.receive. + */ +inline constexpr auto proposalTrusted = makeStr("proposal_trusted"); +inline constexpr auto validationTrusted = makeStr("validation_trusted"); +} // namespace attr + +} // namespace xrpl::telemetry::peer_span diff --git a/src/xrpld/overlay/detail/TrafficCount.h b/src/xrpld/overlay/detail/TrafficCount.h index 8dc8ddb08ee..055f9c270c3 100644 --- a/src/xrpld/overlay/detail/TrafficCount.h +++ b/src/xrpld/overlay/detail/TrafficCount.h @@ -282,7 +282,7 @@ class TrafficCount {Category::ShareCasObject, "getobject_CAS_share"}, {Category::GetCasObject, "getobject_CAS_get"}, {Category::ShareFetchPack, "getobject_Fetch_Pack_share"}, - {Category::GetFetchPack, "getobject_Fetch Pack_get"}, + {Category::GetFetchPack, "getobject_Fetch_Pack_get"}, {Category::GetTransactions, "getobject_Transactions_get"}, {Category::ShareHash, "getobject_share"}, {Category::GetHash, "getobject_get"}, diff --git a/src/xrpld/perflog/detail/PerfLogImp.cpp b/src/xrpld/perflog/detail/PerfLogImp.cpp index 3aa7e38ea25..f43464dd1de 100644 --- a/src/xrpld/perflog/detail/PerfLogImp.cpp +++ b/src/xrpld/perflog/detail/PerfLogImp.cpp @@ -1,6 +1,8 @@ #include #include +#include +#include #include #include @@ -12,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -330,6 +333,17 @@ PerfLogImp::rpcStart(std::string const& method, std::uint64_t const requestId) } std::scoped_lock const lock(counters_.methodsMutex); counters_.methods[requestId] = {counter->first.c_str(), steady_clock::now()}; + + // Task 9.4: Record RPC start in OTel metrics pipeline. + if (auto* mr = app_.getMetricsRegistry()) + mr->recordRpcStarted(method); + + // Proof-of-concept for tasks/metric-macro-plan.md Use Case 2: a value + // that must be able to decrease (UpDownCounter), added at its call + // site with no MetricsRegistry member/init-line/method. Paired with the + // matching -1 in rpcEnd(). Runs on the same path as recordRpcStarted + // above, i.e. only after a methods-map entry exists for this request. + XRPL_METRIC_UPDOWN_ADD(app_, "rpc_in_flight_requests", "RPC requests currently executing", 1); } void @@ -356,24 +370,51 @@ PerfLogImp::rpcEnd(std::string const& method, std::uint64_t const requestId, boo { // LCOV_EXCL_START UNREACHABLE("xrpl::perf::PerfLogImp::rpcEnd : valid requestId input"); + // Without an entry, startTime stays default-initialized; returning + // avoids recording a bogus (now - epoch) duration to the counters + // and OTel histogram below. UNREACHABLE does not halt release builds. + return; // LCOV_EXCL_STOP } } - std::scoped_lock const lock(counter->second.mutex); - if (finish) + auto const durationUs = + std::chrono::duration_cast(steady_clock::now() - startTime); { - ++counter->second.value.finished; + std::scoped_lock const lock(counter->second.mutex); + if (finish) + { + ++counter->second.value.finished; + } + else + { + ++counter->second.value.errored; + } + counter->second.value.duration += durationUs; } - else + + // Task 9.4: Record RPC completion in OTel metrics pipeline. + // Mirrors the rpcStart() instrumentation so the finished/errored + // counters and duration histogram advance with every call. + if (auto* mr = app_.getMetricsRegistry()) { - ++counter->second.value.errored; + if (finish) + { + mr->recordRpcFinished(method, durationUs.count()); + } + else + { + mr->recordRpcErrored(method, durationUs.count()); + } } - counter->second.value.duration += - std::chrono::duration_cast(steady_clock::now() - startTime); + + // Matching -1 for the +1 recorded in rpcStart(). Placed after the early + // returns above so it runs only when this request's methods-map entry was + // found (i.e. a +1 was recorded for it), keeping the in-flight count balanced. + XRPL_METRIC_UPDOWN_ADD(app_, "rpc_in_flight_requests", "RPC requests currently executing", -1); } void -PerfLogImp::jobQueue(JobType const type) +PerfLogImp::jobQueue(JobType const type, std::string const& name) { auto counter = counters_.jq.find(type); if (counter == counters_.jq.end()) @@ -385,11 +426,16 @@ PerfLogImp::jobQueue(JobType const type) } std::scoped_lock const lock(counter->second.mutex); ++counter->second.value.queued; + + // Task 9.5: Record job enqueue in OTel metrics pipeline. + if (auto* mr = app_.getMetricsRegistry()) + mr->recordJobQueued(JobTypes::name(type), name); } void PerfLogImp::jobStart( JobType const type, + std::string const& name, microseconds dur, steady_time_point startTime, int instance) @@ -411,10 +457,14 @@ PerfLogImp::jobStart( std::scoped_lock const lock(counters_.jobsMutex); if (instance >= 0 && instance < counters_.jobs.size()) counters_.jobs[instance] = {type, startTime}; + + // Task 9.5: Record job start in OTel metrics pipeline. + if (auto* mr = app_.getMetricsRegistry()) + mr->recordJobStarted(JobTypes::name(type), name, dur.count()); } void -PerfLogImp::jobFinish(JobType const type, microseconds dur, int instance) +PerfLogImp::jobFinish(JobType const type, std::string const& name, microseconds dur, int instance) { auto counter = counters_.jq.find(type); if (counter == counters_.jq.end()) @@ -433,6 +483,10 @@ PerfLogImp::jobFinish(JobType const type, microseconds dur, int instance) std::scoped_lock const lock(counters_.jobsMutex); if (instance >= 0 && instance < counters_.jobs.size()) counters_.jobs[instance] = {JtInvalid, steady_time_point()}; + + // Task 9.5: Record job finish in OTel metrics pipeline. + if (auto* mr = app_.getMetricsRegistry()) + mr->recordJobFinished(JobTypes::name(type), name, dur.count()); } void diff --git a/src/xrpld/perflog/detail/PerfLogImp.h b/src/xrpld/perflog/detail/PerfLogImp.h index 14477512ff8..9c8f38bb741 100644 --- a/src/xrpld/perflog/detail/PerfLogImp.h +++ b/src/xrpld/perflog/detail/PerfLogImp.h @@ -153,12 +153,16 @@ class PerfLogImp : public PerfLog } void - jobQueue(JobType const type) override; + jobQueue(JobType const type, std::string const& name) override; void - jobStart(JobType const type, microseconds dur, steady_time_point startTime, int instance) - override; + jobStart( + JobType const type, + std::string const& name, + microseconds dur, + steady_time_point startTime, + int instance) override; void - jobFinish(JobType const type, microseconds dur, int instance) override; + jobFinish(JobType const type, std::string const& name, microseconds dur, int instance) override; json::Value countersJson() const override diff --git a/src/xrpld/rpc/ServerHandler.h b/src/xrpld/rpc/ServerHandler.h index 054bec9b5b2..fbe487654ee 100644 --- a/src/xrpld/rpc/ServerHandler.h +++ b/src/xrpld/rpc/ServerHandler.h @@ -192,7 +192,12 @@ class ServerHandler void processSession(std::shared_ptr const&, std::shared_ptr coro); - void + /** + * Process an RPC request and write the reply to `output`. + * @return false if the request resulted in an error response, true + * otherwise. Lets the caller's enclosing span reflect the outcome. + */ + bool processRequest( Port const& port, std::string const& request, diff --git a/src/xrpld/rpc/detail/PathFindSpanNames.h b/src/xrpld/rpc/detail/PathFindSpanNames.h new file mode 100644 index 00000000000..65ce81bb964 --- /dev/null +++ b/src/xrpld/rpc/detail/PathFindSpanNames.h @@ -0,0 +1,114 @@ +#pragma once + +/** + * Compile-time span name constants for PathFind tracing. + * + * Covers the path_find and ripple_path_find RPC handlers, the + * PathRequest computation engine, and the Pathfinder graph exploration. + * + * Span hierarchy: + * + * RPC entry (one-shot or subscription): + * + * +----------------------------------------------------------------+ + * | pathfind.request | + * | doPathFind() / doRipplePathFind() | + * | attrs: pathfind_source_account, pathfind_dest_account | + * | (set when present in request params) | + * | | + * | +-----------------------------------------------------------+ | + * | | pathfind.compute | | + * | | PathRequest::doUpdate() | | + * | | attrs: pathfind_fast | | + * | | | | + * | | +-----------------------------------------------------+ | | + * | | | pathfind.discover (one per RPC call, hoisted above | | + * | | | the per-source-asset loop in PathRequest::findPaths)| | + * | | | attrs: pathfind_search_level, pathfind_num_paths | | + * | | +-----------------------------------------------------+ | | + * | +-----------------------------------------------------------+ | + * +----------------------------------------------------------------+ + * + * Async recomputation (ledger close): + * + * +----------------------------------------------------------------+ + * | pathfind.update_all | + * | PathRequestManager::updateAll() | + * | attrs: pathfind_ledger_index, pathfind_num_requests | + * | | + * | +-----------------------------------------------------------+ | + * | | pathfind.compute (per active request) | | + * | +-----------------------------------------------------------+ | + * +----------------------------------------------------------------+ + */ + +#include + +namespace xrpl::telemetry::pathfind_span { + +// ===== Span prefixes ======================================================= + +namespace prefix { +/** + * "pathfind" — root prefix for path finding spans. + */ +inline constexpr auto pathfind = makeStr("pathfind"); +} // namespace prefix + +// ===== Span operation suffixes ============================================= + +namespace op { +inline constexpr auto request = makeStr("request"); +inline constexpr auto compute = makeStr("compute"); +inline constexpr auto updateAll = makeStr("update_all"); +inline constexpr auto discover = makeStr("discover"); +} // namespace op + +// ===== Attribute keys ====================================================== +// +// All pathfind attributes are namespaced under `pathfind_*` (underscore form, +// per Phase 1c naming spec rule 5). Avoids collisions with bare keys like +// `fast` or `num_paths` that other subsystems may introduce. + +namespace attr { +/** + * "pathfind_source_account" — originating account for path search. + */ +inline constexpr auto sourceAccount = makeStr("pathfind_source_account"); +/** + * "pathfind_dest_account" — destination account. + */ +inline constexpr auto destAccount = makeStr("pathfind_dest_account"); +/** + * "pathfind_fast" — whether fast pathfinding mode enabled. + */ +inline constexpr auto fast = makeStr("pathfind_fast"); +/** + * "pathfind_search_level" — depth of graph exploration. + */ +inline constexpr auto searchLevel = makeStr("pathfind_search_level"); +/** + * "pathfind_num_paths" — total paths produced across the per-source-asset + * loop in PathRequest::findPaths (sum of getBestPaths().size() per asset). + */ +inline constexpr auto numPaths = makeStr("pathfind_num_paths"); +/** + * "pathfind_num_requests" — snapshot size of requests_ at update_all start + * (may include weak_ptrs that subsequently expire during processing). + */ +inline constexpr auto numRequests = makeStr("pathfind_num_requests"); +/** + * "pathfind_ledger_index" — pathfind target ledger index. + */ +inline constexpr auto ledgerIndex = makeStr("pathfind_ledger_index"); +/** + * "pathfind_dest_currency" — destination currency code. + */ +inline constexpr auto destCurrency = makeStr("pathfind_dest_currency"); +/** + * "pathfind_num_source_assets" — candidate source assets count. + */ +inline constexpr auto numSourceAssets = makeStr("pathfind_num_source_assets"); +} // namespace attr + +} // namespace xrpl::telemetry::pathfind_span diff --git a/src/xrpld/rpc/detail/PathRequest.cpp b/src/xrpld/rpc/detail/PathRequest.cpp index 3c09917dad0..28a6ad3c5ca 100644 --- a/src/xrpld/rpc/detail/PathRequest.cpp +++ b/src/xrpld/rpc/detail/PathRequest.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -34,10 +35,12 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -579,6 +582,22 @@ PathRequest::findPaths( auto const dstAmount = convertAmount(saDstAmount_, convertAll_); hash_map> currencyMap; + + // One `pathfind.discover` span wraps the entire per-source-asset loop so + // that a single RPC call produces one discover span instead of N (one per + // candidate source asset). Trade-off: per-asset discovery/ranking timing + // is no longer split into individual spans — span count and Tempo storage + // are bounded per RPC at the cost of per-asset visibility. If per-asset + // breakdown is needed in the future, add child spans inside the loop body + // (`Pathfinder::findPaths`/`computePathRanks`) parented off this span. + using namespace telemetry; + auto span = SpanGuard::span( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::discover); + span.setAttribute(pathfind_span::attr::searchLevel, static_cast(level)); + span.setAttribute( + pathfind_span::attr::numSourceAssets, static_cast(sourceAssets.size())); + + std::int64_t totalPaths = 0; for (auto const& asset : sourceAssets) { if (continueCallback && !continueCallback()) @@ -598,6 +617,7 @@ PathRequest::findPaths( auto ps = pathfinder->getBestPaths( kMaxPaths, fullLiquidityPath, context_[asset], asset.getIssuer(), continueCallback); context_[asset] = ps; + totalPaths += static_cast(ps.size()); auto const& sourceAccount = [&] { if (!isXRP(asset.getIssuer())) @@ -700,6 +720,8 @@ PathRequest::findPaths( } } + span.setAttribute(pathfind_span::attr::numPaths, totalPaths); + /* The resource fee is based on the number of source currencies used. The minimum cost is 50 and the maximum is 400. The cost increases after four source currencies, 50 - (4 * 4) = 34. @@ -716,6 +738,14 @@ PathRequest::doUpdate( std::function const& continueCallback) { using namespace std::chrono; + using namespace telemetry; + // Scoped so pathfind.discover (created synchronously below via findPaths) + // nests under it. doUpdate does not yield, so scoping is safe. + auto span = ScopedSpanGuard( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::compute); + span.setAttribute(pathfind_span::attr::fast, fast); + span.setAttribute(pathfind_span::attr::destCurrency, to_string(saDstAmount_.asset()).c_str()); + JLOG(journal_.debug()) << iIdentifier_ << " update " << (fast ? "fast" : "normal"); { diff --git a/src/xrpld/rpc/detail/PathRequestManager.cpp b/src/xrpld/rpc/detail/PathRequestManager.cpp index 4953634181e..9141aaded48 100644 --- a/src/xrpld/rpc/detail/PathRequestManager.cpp +++ b/src/xrpld/rpc/detail/PathRequestManager.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include @@ -72,6 +74,19 @@ PathRequestManager::updateAll(std::shared_ptr const& inLedger) cache = getAssetCache(inLedger, true); } + using namespace telemetry; + // updateAll runs on every ledger close. Skip span emission when there are + // no active path subscriptions, to avoid a steady stream of empty spans at + // mainnet close cadence. A null guard is used in that case; all other work + // still runs unchanged (notably the isNewPathRequest() flag reset below), + // so behaviour matches the pre-span code path. + auto span = requests.empty() + ? SpanGuard{} + : SpanGuard::span( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::updateAll); + span.setAttribute(pathfind_span::attr::ledgerIndex, static_cast(inLedger->seq())); + span.setAttribute(pathfind_span::attr::numRequests, static_cast(requests.size())); + bool newRequests = app_.getLedgerMaster().isNewPathRequest(); bool mustBreak = false; diff --git a/src/xrpld/rpc/detail/RPCHandler.cpp b/src/xrpld/rpc/detail/RPCHandler.cpp index 6f46aed62dd..208b0ff49db 100644 --- a/src/xrpld/rpc/detail/RPCHandler.cpp +++ b/src/xrpld/rpc/detail/RPCHandler.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -17,14 +18,18 @@ #include #include #include +#include #include #include #include #include #include +#include -namespace xrpl::RPC { +namespace xrpl { +using namespace telemetry; +namespace RPC { namespace { @@ -157,6 +162,17 @@ template Status callMethod(JsonContext& context, Method method, std::string const& name, Object& result) { + // Scoped so this command nests under rpc.process and becomes the ambient + // parent of any command-internal spans (e.g. pathfind.request). Coro-aware + // storage keeps the scope correct across doRipplePathFind's yield. + auto span = ScopedSpanGuard(TraceCategory::Rpc, rpc_span::prefix::command, name); + span.setAttribute(rpc_span::attr::command, name.c_str()); + span.setAttribute(rpc_span::attr::version, static_cast(context.apiVersion)); + span.setAttribute( + rpc_span::attr::rpcRole, + context.role == Role::ADMIN ? std::string_view(rpc_span::val::admin) + : std::string_view(rpc_span::val::user)); + static std::atomic kRequestId{0}; auto& perfLog = context.app.getPerfLog(); std::uint64_t const curId = ++kRequestId; @@ -172,12 +188,32 @@ callMethod(JsonContext& context, Method method, std::string const& name, Object& JLOG(context.j.debug()) << "RPC call " << name << " completed in " << ((end - start).count() / 1000000000.0) << "seconds"; perfLog.rpcFinish(name, curId); + span.setAttribute(rpc_span::attr::loadType, context.loadType.label().c_str()); + // Status::operator bool() returns true when there IS an error + // (code_ != OK), so the ternary correctly maps error->error, ok->success. + span.setAttribute( + rpc_span::attr::rpcStatus, + ret ? std::string_view{rpc_span::val::error} + : std::string_view{rpc_span::val::success}); + // Reflect the result in the OTel span status, not just the attribute, + // so non-exception RPC errors (rpcTOO_BUSY, rpcNO_PERMISSION, ...) are + // visible to {status.code=error} queries. + if (ret) + { + span.setError(rpc_span::val::error); + } + else + { + span.setOk(); + } return ret; } catch (std::exception& e) { perfLog.rpcError(name, curId); JLOG(context.j.info()) << "Caught throw: " << e.what(); + span.recordException(e); + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); if (context.loadType == Resource::kFeeReferenceRpc) context.loadType = Resource::kFeeExceptionRpc; @@ -187,6 +223,31 @@ callMethod(JsonContext& context, Method method, std::string const& name, Object& } } +// Resolve the span suffix / command attribute for a request that failed in +// fillHandler. Returns the canonical handler name for a recognized command +// (a finite, bounded set) or the literal "unknown" for a request that omits +// both fields or names an unregistered command. The raw request value is +// deliberately NOT used: the command attribute is promoted to a Prometheus +// label by the spanmetrics connector, so an attacker-controlled string would +// let arbitrary request input drive unbounded span-name / label cardinality. +// Resolving against the registry keeps per-command error attribution for real +// commands (e.g. a submit rejected with rpcTOO_BUSY stays rpc.command.submit) +// while collapsing garbage input to a single series. +std::string_view +resolveCommandSpanName(JsonContext const& context) +{ + if (!context.params.isMember(jss::command) && !context.params.isMember(jss::method)) + return rpc_span::val::unknownCommand; + + std::string const cmd = context.params.isMember(jss::command) + ? context.params[jss::command].asString() + : context.params[jss::method].asString(); + + auto const* handler = getHandler(context.apiVersion, context.app.config().betaRpcApi, cmd); + return (handler != nullptr) ? std::string_view{handler->name} + : std::string_view{rpc_span::val::unknownCommand}; +} + } // namespace Status @@ -195,6 +256,15 @@ doCommand(RPC::JsonContext& context, json::Value& result) Handler const* handler = nullptr; if (auto error = fillHandler(context, handler)) { + // Bound the span name and command attribute to the finite set of + // registered handler names (plus "unknown") — see the helper for why + // raw request input must not reach the telemetry pipeline. + auto const cmdName = resolveCommandSpanName(context); + auto span = ScopedSpanGuard(TraceCategory::Rpc, rpc_span::prefix::command, cmdName); + span.setAttribute(rpc_span::attr::command, cmdName); + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); + span.setError(getErrorInfo(error).token.cStr()); + injectError(error, result); return error; } @@ -234,4 +304,5 @@ roleRequired(unsigned int version, bool betaEnabled, std::string const& method) return handler->role; } -} // namespace xrpl::RPC +} // namespace RPC +} // namespace xrpl diff --git a/src/xrpld/rpc/detail/RpcSpanNames.h b/src/xrpld/rpc/detail/RpcSpanNames.h new file mode 100644 index 00000000000..07202a7592e --- /dev/null +++ b/src/xrpld/rpc/detail/RpcSpanNames.h @@ -0,0 +1,190 @@ +#pragma once + +/** + * Compile-time span name constants for the RPC subsystem. + * + * All span prefixes, operation names, and attribute keys used by RPC + * tracing call sites are defined here. Built on the StaticStr/join() + * primitives from . + * + * Usage: + * @code + * #include + * using namespace telemetry; + * + * auto span = SpanGuard::span( + * TraceCategory::Rpc, rpc_span::prefix::command, "submit"); + * span.setAttribute(rpc_span::attr::command, "submit"); + * span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::success); + * @endcode + * + * Span hierarchy (automatic nesting via OTel thread-local context): + * + * HTTP JSON-RPC path (single request): + * + * +-------------------------------------------------------+ + * | rpc.http_request | + * | ServerHandler::processSession(Session) | + * | | + * | +--------------------------------------------------+ | + * | | rpc.process | | + * | | ServerHandler::processRequest() | | + * | | | | + * | | +---------------------------------------------+ | | + * | | | rpc.command.{name} | | | + * | | | RPC::callMethod() | | | + * | | | attrs: command, version, rpc_role, rpc_status | | | + * | | +---------------------------------------------+ | | + * | +--------------------------------------------------+ | + * +-------------------------------------------------------+ + * + * HTTP batch path (multiple commands per request): + * + * +-------------------------------------------------------+ + * | rpc.http_request | + * | | + * | +--------------------------------------------------+ | + * | | rpc.process | | + * | | | | + * | | +------------------+ +------------------+ | | + * | | | rpc.command.{a} | | rpc.command.{b} | ... | | + * | | +------------------+ +------------------+ | | + * | +--------------------------------------------------+ | + * +-------------------------------------------------------+ + * + * WebSocket path: + * + * +-------------------------------------------------------+ + * | rpc.ws_message | + * | ServerHandler::processSession(WSSession) | + * | | + * | +--------------------------------------------------+ | + * | | rpc.command.{name} | | + * | | RPC::callMethod() | | + * | | attrs: command, version, rpc_role, rpc_status | | + * | +--------------------------------------------------+ | + * +-------------------------------------------------------+ + * + * WebSocket error paths: + * + * +-------------------------------------------------------+ + * | rpc.ws_message (error: invalid_json) | + * | ServerHandler::onWSMessage() — parse failure | + * +-------------------------------------------------------+ + * + * +-------------------------------------------------------+ + * | rpc.ws_upgrade | + * | ServerHandler::onHandoff() — upgrade try/catch | + * +-------------------------------------------------------+ + * + * Command dispatch error path: + * + * +-------------------------------------------------------+ + * | rpc.command.{name} (error: too_busy/unknown/etc) | + * | RPC::doCommand() — fillHandler() rejection | + * +-------------------------------------------------------+ + * + * gRPC path (see GrpcSpanNames.h for constants): + * + * +-------------------------------------------------------+ + * | grpc. (e.g. grpc.GetLedger) | + * | CallData::process(coro) | + * | attrs: method, grpc_status | + * +-------------------------------------------------------+ + * + * Covered paths: + * - HTTP JSON-RPC (single and batch requests) + * - WebSocket RPC commands + * - WebSocket message parse errors (invalid JSON, oversized) + * - WebSocket upgrade failures (protocol handshake errors) + * - Admin CLI (connects via HTTP internally) + * - Command dispatch rejections (unknown cmd, too busy, no perm) + * - gRPC endpoints (GetLedger, GetLedgerData, GetLedgerDiff, + * GetLedgerEntry) + * - Command execution: timing, success/failure, exceptions + * - Per-command attributes: name, API version, rpc_role, rpc_status + * + * Known gaps (not yet instrumented): + * - Early validation errors in processRequest() before rpc.process + * span (malformed JSON, auth failures, oversized requests) + * - Subscription push notifications (server-initiated, not RPC) + */ + +#include + +namespace xrpl::telemetry::rpc_span { + +// ===== Span prefixes ======================================================= + +namespace prefix { +/** + * "rpc" — root prefix for transport-level spans. + */ +inline constexpr auto rpc = seg::rpc; +/** + * "rpc.command" — prefix for individual RPC command spans. + */ +inline constexpr auto command = join(seg::rpc, makeStr("command")); +} // namespace prefix + +// ===== Span operation suffixes ============================================= + +namespace op { +inline constexpr auto wsMessage = makeStr("ws_message"); +inline constexpr auto wsUpgrade = makeStr("ws_upgrade"); +inline constexpr auto httpRequest = makeStr("http_request"); +inline constexpr auto process = makeStr("process"); +} // namespace op + +// ===== Attribute keys ====================================================== + +namespace attr { +/** + * "command" — RPC method name. + */ +inline constexpr auto command = makeStr("command"); +/** + * "version" — api_version per request. + */ +inline constexpr auto version = makeStr("version"); +/** + * "rpc_role" — admin|user. Domain-qualified: collides with grpc_role. + */ +inline constexpr auto rpcRole = makeStr("rpc_role"); +/** + * "rpc_status" — success|error. Domain-qualified: avoids OTel reserved span status. + */ +inline constexpr auto rpcStatus = makeStr("rpc_status"); +/** + * "request_payload_size" — bytes of inbound request payload. + */ +inline constexpr auto requestPayloadSize = makeStr("request_payload_size"); +/** + * "is_batch" — whether request is a JSON-RPC batch. + */ +inline constexpr auto isBatch = makeStr("is_batch"); +/** + * "batch_size" — number of sub-requests in a batch. + */ +inline constexpr auto batchSize = makeStr("batch_size"); +/** + * "load_type" — resource cost category after execution. + */ +inline constexpr auto loadType = makeStr("load_type"); +} // namespace attr + +// ===== Attribute values ==================================================== + +namespace val { +using telemetry::attr_val::error; +using telemetry::attr_val::success; +inline constexpr auto admin = makeStr("admin"); +inline constexpr auto user = makeStr("user"); +inline constexpr auto unknownCommand = makeStr("unknown"); +/** + * "invalid_json" — WS message parse failure or oversize. + */ +inline constexpr auto invalidJson = makeStr("invalid_json"); +} // namespace val + +} // namespace xrpl::telemetry::rpc_span diff --git a/src/xrpld/rpc/detail/ServerHandler.cpp b/src/xrpld/rpc/detail/ServerHandler.cpp index 768cbf0dc07..2c7929a5175 100644 --- a/src/xrpld/rpc/detail/ServerHandler.cpp +++ b/src/xrpld/rpc/detail/ServerHandler.cpp @@ -4,8 +4,10 @@ #include #include #include +#include #include #include +#include // IWYU pragma: keep #include #include @@ -43,6 +45,7 @@ #include #include #include +#include #include #include @@ -60,6 +63,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +75,7 @@ #include namespace xrpl { +using namespace telemetry; class Peer; class LedgerMaster; @@ -224,13 +229,19 @@ ServerHandler::onHandoff( if (!isWs) return statusRequestResponse(request, http::status::unauthorized); + // Fresh root so each WS upgrade is its own trace, not nested under a + // leaked ambient span on a reused coro worker. + auto span = ScopedSpanGuard::freshRoot( + TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::wsUpgrade); std::shared_ptr ws; try { ws = session.websocketUpgrade(); + span.setOk(); } catch (std::exception const& e) { + span.recordException(e); // LCOV_EXCL_LINE JLOG(journal_.error()) << "Exception upgrading websocket: " << e.what() << "\n"; return statusRequestResponse(request, http::status::internal_server_error); } @@ -339,10 +350,15 @@ ServerHandler::onWSMessage( auto const size = boost::asio::buffer_size(buffers); if (size > RPC::Tuning::kMaxRequestSize || !json::Reader{}.parse(jv, buffers) || !jv.isObject()) { + // Fresh root so each WS message is its own trace. + auto span = ScopedSpanGuard::freshRoot( + TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::wsMessage); + span.setError(rpc_span::val::invalidJson); + json::Value jvResult(json::ValueType::Object); jvResult[jss::type] = jss::error; jvResult[jss::error] = "jsonInvalid"; - jvResult[jss::value] = buffersToString(buffers); + jvResult[jss::value] = ::xrpl::buffersToString(buffers); boost::beast::multi_buffer sb; json::stream(jvResult, [&sb](auto const p, auto const n) { sb.commit(boost::asio::buffer_copy(sb.prepare(n), boost::asio::buffer(p, n))); @@ -415,12 +431,26 @@ ServerHandler::processSession( std::shared_ptr const& coro, json::Value const& jv) { + // Fresh root so each WS message is its own trace. + auto span = ScopedSpanGuard::freshRoot( + TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::wsMessage); + if (jv.isMember(jss::command) && jv[jss::command].isString()) + { + span.setAttribute(rpc_span::attr::command, jv[jss::command].asString().c_str()); + } + else if (jv.isMember(jss::method) && jv[jss::method].isString()) + { + span.setAttribute(rpc_span::attr::command, jv[jss::method].asString().c_str()); + } + auto is = std::static_pointer_cast(session->appDefined); if (is->getConsumer().disconnect(journal_)) { session->close({boost::beast::websocket::policy_error, "threshold exceeded"}); // FIX: This rpcError is not delivered since the session // was just closed. + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); + span.setError("resource threshold exceeded"); return rpcError(RpcSlowDown); } @@ -452,6 +482,8 @@ ServerHandler::processSession( jr[jss::api_version] = jv[jss::api_version]; is->getConsumer().charge(Resource::kFeeMalformedRpc); + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); + span.setError(jr[jss::error].asString()); return jr; } @@ -498,6 +530,8 @@ ServerHandler::processSession( jr[jss::result] = RPC::makeError(RpcInternal); JLOG(journal_.error()) << "Exception while processing WS: " << ex.what() << "\n" << "Input JSON: " << json::Compact{json::Value{jv}}; + span.recordException(ex); + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); // LCOV_EXCL_STOP } @@ -530,12 +564,19 @@ ServerHandler::processSession( } jr[jss::request] = rq; + // Mark the span according to the final result. Doing it here (rather + // than an unconditional setOk later) ensures error responses — from + // doCommand, a FORBID role, or the catch block above — are not + // overwritten as OK. + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); + span.setError(jr[jss::error].asString()); } else { if (jr[jss::result].isMember("forwarded") && jr[jss::result]["forwarded"]) jr = jr[jss::result]; jr[jss::status] = jss::success; + span.setOk(); } if (jv.isMember(jss::id)) @@ -557,9 +598,17 @@ ServerHandler::processSession( std::shared_ptr const& session, std::shared_ptr coro) { - processRequest( + // Fresh root so each HTTP request is its own trace, not nested under a + // leaked ambient span on a reused coro worker. + auto span = ScopedSpanGuard::freshRoot( + TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::httpRequest); + + auto const requestBody = ::xrpl::buffersToString(session->request().body().data()); + span.setAttribute(rpc_span::attr::requestPayloadSize, static_cast(requestBody.size())); + + bool const ok = processRequest( session->port(), - buffersToString(session->request().body().data()), + requestBody, session->remoteAddress().atPort(0), makeOutput(*session), coro, @@ -579,6 +628,15 @@ ServerHandler::processSession( { session->close(true); } + // Reflect the request outcome on the wrapper span instead of always OK. + if (ok) + { + span.setOk(); + } + else + { + span.setError(rpc_span::val::error); + } } static json::Value @@ -597,7 +655,7 @@ constexpr json::Int kServerOverloaded = -32604; constexpr json::Int kForbidden = -32605; constexpr json::Int kWrongVersion = -32606; -void +bool ServerHandler::processRequest( Port const& port, std::string const& request, @@ -607,20 +665,37 @@ ServerHandler::processRequest( std::string_view forwardedFor, std::string_view user) { + // Scoped child of rpc.http_request. Safe to hold across the coroutine + // yield in doRipplePathFind: the coro-aware context storage moves this + // scope with the coroutine on resume (it is never stranded on a worker's + // thread-local stack), so nesting and log-trace correlation both hold. + auto span = ScopedSpanGuard(TraceCategory::Rpc, rpc_span::prefix::rpc, rpc_span::op::process); auto rpcJ = app_.getJournal("RPC"); + // Tracks whether any failure occurred. Set on every error path (early + // returns, the catch block, and per-request error replies) and used at the + // end to mark the span status. The HTTP status code alone is insufficient: + // it stays 200 for batch responses and for ripplerpc < 3.0, so relying on + // it would let payload-level errors end the span as successful. + bool spanHadError = false; + + // Marks the span as failed before sending an error reply, so the + // early-return validation paths below are not later seen as successful + // (the span would otherwise end UNSET, invisible to {status.code=error}). + auto httpReplyError = [&](int status, std::string const& message) { + spanHadError = true; + span.setError(message); + httpReply(status, message, output, rpcJ); + }; + json::Value jsonOrig; { json::Reader reader; if ((request.size() > RPC::Tuning::kMaxRequestSize) || !reader.parse(request, jsonOrig) || !jsonOrig || !jsonOrig.isObject()) { - httpReply( - 400, - "Unable to parse request: " + reader.getFormattedErrorMessages(), - output, - rpcJ); - return; + httpReplyError(400, "Unable to parse request: " + reader.getFormattedErrorMessages()); + return false; } } @@ -631,11 +706,14 @@ ServerHandler::processRequest( batch = true; if (!jsonOrig.isMember(jss::params) || !jsonOrig[jss::params].isArray()) { - httpReply(400, "Malformed batch request", output, rpcJ); - return; + httpReplyError(400, "Malformed batch request"); + return false; } size = jsonOrig[jss::params].size(); } + span.setAttribute(rpc_span::attr::isBatch, batch); + if (batch) + span.setAttribute(rpc_span::attr::batchSize, static_cast(size)); json::Value reply(batch ? json::ValueType::Array : json::ValueType::Object); auto const start(std::chrono::high_resolution_clock::now()); @@ -670,8 +748,8 @@ ServerHandler::processRequest( { if (!batch) { - httpReply(400, jss::invalid_API_version.cStr(), output, rpcJ); - return; + httpReplyError(400, jss::invalid_API_version.cStr()); + return false; } json::Value r(json::ValueType::Object); r[jss::request] = jsonRPC; @@ -713,8 +791,8 @@ ServerHandler::processRequest( { if (!batch) { - httpReply(503, "Server is overloaded", output, rpcJ); - return; + httpReplyError(503, "Server is overloaded"); + return false; } json::Value r = jsonRPC; r[jss::error] = makeJsonError(kServerOverloaded, "Server is overloaded"); @@ -728,8 +806,8 @@ ServerHandler::processRequest( usage.charge(Resource::kFeeMalformedRpc); if (!batch) { - httpReply(403, "Forbidden", output, rpcJ); - return; + httpReplyError(403, "Forbidden"); + return false; } json::Value r = jsonRPC; r[jss::error] = makeJsonError(kForbidden, "Forbidden"); @@ -742,8 +820,8 @@ ServerHandler::processRequest( usage.charge(Resource::kFeeMalformedRpc); if (!batch) { - httpReply(400, "Null method", output, rpcJ); - return; + httpReplyError(400, "Null method"); + return false; } json::Value r = jsonRPC; r[jss::error] = makeJsonError(kMethodNotFound, "Null method"); @@ -757,8 +835,8 @@ ServerHandler::processRequest( usage.charge(Resource::kFeeMalformedRpc); if (!batch) { - httpReply(400, "method is not string", output, rpcJ); - return; + httpReplyError(400, "method is not string"); + return false; } json::Value r = jsonRPC; r[jss::error] = makeJsonError(kMethodNotFound, "method is not string"); @@ -772,8 +850,8 @@ ServerHandler::processRequest( usage.charge(Resource::kFeeMalformedRpc); if (!batch) { - httpReply(400, "method is empty", output, rpcJ); - return; + httpReplyError(400, "method is empty"); + return false; } json::Value r = jsonRPC; r[jss::error] = makeJsonError(kMethodNotFound, "method is empty"); @@ -798,8 +876,8 @@ ServerHandler::processRequest( else if (!params.isArray() || params.size() != 1) { usage.charge(Resource::kFeeMalformedRpc); - httpReply(400, "params unparsable", output, rpcJ); - return; + httpReplyError(400, "params unparsable"); + return false; } else { @@ -807,8 +885,8 @@ ServerHandler::processRequest( if (!params.isObjectOrNull()) { usage.charge(Resource::kFeeMalformedRpc); - httpReply(400, "params unparsable", output, rpcJ); - return; + httpReplyError(400, "params unparsable"); + return false; } } } @@ -825,8 +903,8 @@ ServerHandler::processRequest( usage.charge(Resource::kFeeMalformedRpc); if (!batch) { - httpReply(400, "ripplerpc is not a string", output, rpcJ); - return; + httpReplyError(400, "ripplerpc is not a string"); + return false; } json::Value r = jsonRPC; @@ -883,6 +961,9 @@ ServerHandler::processRequest( JLOG(journal_.error()) << "Internal error : " << ex.what() << " when processing request: " << json::Compact{json::Value{params}}; + span.recordException(ex); + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); + spanHadError = true; // LCOV_EXCL_STOP } @@ -899,6 +980,7 @@ ServerHandler::processRequest( { if (result.isMember(jss::error)) { + spanHadError = true; result[jss::status] = jss::error; result["code"] = result[jss::error_code]; result["message"] = result[jss::error_message]; @@ -919,6 +1001,7 @@ ServerHandler::processRequest( // received. if (result.isMember(jss::error)) { + spanHadError = true; auto rq = params; if (rq.isObject()) @@ -1014,7 +1097,20 @@ ServerHandler::processRequest( } } + // Mark the span error if any request failed or the HTTP status is an error. + // spanHadError catches payload-level errors that httpStatus misses (batch + // responses and ripplerpc < 3.0 always return HTTP 200). + if (spanHadError || httpStatus >= 400) + { + span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); + span.setError(rpc_span::val::error); + } + else + { + span.setOk(); + } httpReply(httpStatus, response, output, rpcJ); + return !spanHadError; } //------------------------------------------------------------------------------ diff --git a/src/xrpld/rpc/handlers/orderbook/PathFind.cpp b/src/xrpld/rpc/handlers/orderbook/PathFind.cpp index 3a6b52ee985..bf7fdfe2646 100644 --- a/src/xrpld/rpc/handlers/orderbook/PathFind.cpp +++ b/src/xrpld/rpc/handlers/orderbook/PathFind.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -9,12 +10,25 @@ #include #include #include +#include +#include namespace xrpl { json::Value doPathFind(RPC::JsonContext& context) { + using namespace telemetry; + // Scoped so pathfind.compute/discover (created synchronously below on this + // thread) nest under it. doPathFind does not yield, so scoping is safe. + auto span = ScopedSpanGuard( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::request); + // Addresses are hashed before emission for privacy. + if (auto const& src = context.params[jss::source_account]; src.isString()) + span.setAttribute(pathfind_span::attr::sourceAccount, redactAccount(src.asString())); + if (auto const& dst = context.params[jss::destination_account]; dst.isString()) + span.setAttribute(pathfind_span::attr::destAccount, redactAccount(dst.asString())); + if (context.app.config().pathSearchMax == 0) return rpcError(RpcNotSupported); diff --git a/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp b/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp index b7edfb6dbe3..f65a41a911a 100644 --- a/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp +++ b/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -13,6 +14,8 @@ #include #include #include +#include +#include #include #include @@ -23,6 +26,20 @@ namespace xrpl { json::Value doRipplePathFind(RPC::JsonContext& context) { + using namespace telemetry; + // pathfind.request nests under rpc.command. This scope is held across + // context.coro->yield() below; the coro-aware OTel context storage moves it + // with the coroutine on resume (it is never stranded on a worker's + // thread-local stack), so the scope pops on the correct store and the + // span's log lines stay trace-correlated. + auto span = ScopedSpanGuard( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::request); + // Addresses are hashed before emission for privacy. + if (auto const& src = context.params[jss::source_account]; src.isString()) + span.setAttribute(pathfind_span::attr::sourceAccount, redactAccount(src.asString())); + if (auto const& dst = context.params[jss::destination_account]; dst.isString()) + span.setAttribute(pathfind_span::attr::destAccount, redactAccount(dst.asString())); + if (context.app.config().pathSearchMax == 0) return rpcError(RpcNotSupported); diff --git a/src/xrpld/shamap/NodeFamily.cpp b/src/xrpld/shamap/NodeFamily.cpp index 2e48117d6a9..77e23d54c92 100644 --- a/src/xrpld/shamap/NodeFamily.cpp +++ b/src/xrpld/shamap/NodeFamily.cpp @@ -27,7 +27,7 @@ NodeFamily::NodeFamily(Application& app, CollectorManager& cm) , j_(app.getJournal("NodeFamily")) , fbCache_( std::make_shared( - "Node family full below cache", + "Node_family_full_below_cache", stopwatch(), app.getJournal("NodeFamilyFulLBelowCache"), cm.collector(), @@ -35,7 +35,7 @@ NodeFamily::NodeFamily(Application& app, CollectorManager& cm) kFullBelowExpiration)) , tnCache_( std::make_shared( - "Node family tree node cache", + "Node_family_tree_node_cache", app.config().getValueFor(SizedItem::TreeCacheSize), std::chrono::seconds(app.config().getValueFor(SizedItem::TreeCacheAge)), stopwatch(), diff --git a/src/xrpld/telemetry/ConsensusReceiveTracing.h b/src/xrpld/telemetry/ConsensusReceiveTracing.h new file mode 100644 index 00000000000..5b7582e13b1 --- /dev/null +++ b/src/xrpld/telemetry/ConsensusReceiveTracing.h @@ -0,0 +1,130 @@ +#pragma once + +/** + * Helper functions for creating consensus receive trace spans. + * + * Encapsulates the logic for creating SpanGuard instances for incoming + * proposal and validation messages with optional protobuf parent + * extraction. When the incoming message carries a TraceContext with a + * valid span_id, the receive span is created as a child of the + * sender's span, enabling cross-node trace correlation. + * + * Dependency diagram: + * + * protocol::TMProposeSet / TMValidation + * | + * v + * proposalReceiveSpan() / validationReceiveSpan() + * | + * +--- has trace_context? ----+ + * | yes | no + * v v + * SpanGuard::hashSpan() with SpanGuard::freshRoot() + * extracted parent context (fresh trace root) + * + * When XRPL_ENABLE_TELEMETRY is not defined, the functions return + * no-op SpanGuard instances (zero overhead, zero dependencies). + * + * Usage: + * @code + * // In PeerImp::onMessage(TMProposeSet): + * auto span = telemetry::proposalReceiveSpan(*m); + * span.setAttribute(...); + * @endcode + * + * @note Span names come from the canonical constants in + * ConsensusSpanNames.h (consensus::span::proposalReceive / + * validationReceive) so they stay in sync with the rest of Phase 4. + */ + +#include + +#include +#include +#include +#include + +#include + +namespace xrpl::telemetry { + +/** + * Create a "consensus.proposal.receive" span for an incoming proposal. + * + * If the message carries a TraceContext with a valid span_id, the + * receive span is created with the sender's context as parent. + * Otherwise a standalone span is created. + * + * @param msg The incoming TMProposeSet protobuf message. + * @return An active SpanGuard, or a null guard if tracing is disabled. + */ +inline SpanGuard +proposalReceiveSpan([[maybe_unused]] protocol::TMProposeSet const& msg) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (msg.has_trace_context()) + { + auto const& tc = msg.trace_context(); + // Reject malformed or all-zero ids from the peer before trusting + // them as a parent. See TraceContextValidation.h. + if (isValidTraceContext(tc)) + { + // Create a child span using the sender's trace_id and + // span_id as parent. Use hashSpan with the sender's + // trace_id so the receiving span shares the same trace. + return SpanGuard::hashSpan( + TraceCategory::Consensus, + consensus::span::proposalReceive, + reinterpret_cast(tc.trace_id().data()), + tc.trace_id().size(), + reinterpret_cast(tc.span_id().data()), + tc.span_id().size(), + tc.has_trace_flags() ? static_cast(tc.trace_flags()) + : std::uint8_t{0}); + } + } +#endif + // No propagated context — start a fresh trace root (not an ambient child). + return SpanGuard::freshRoot( + TraceCategory::Consensus, seg::consensus, consensus::span::op::proposalReceive); +} + +/** + * Create a "consensus.validation.receive" span for an incoming validation. + * + * If the message carries a TraceContext with a valid span_id, the + * receive span is created with the sender's context as parent. + * Otherwise a standalone span is created. + * + * @param msg The incoming TMValidation protobuf message. + * @return An active SpanGuard, or a null guard if tracing is disabled. + */ +inline SpanGuard +validationReceiveSpan([[maybe_unused]] protocol::TMValidation const& msg) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (msg.has_trace_context()) + { + auto const& tc = msg.trace_context(); + // Reject malformed or all-zero ids from the peer before trusting + // them as a parent. See TraceContextValidation.h. + if (isValidTraceContext(tc)) + { + return SpanGuard::hashSpan( + TraceCategory::Consensus, + consensus::span::validationReceive, + reinterpret_cast(tc.trace_id().data()), + tc.trace_id().size(), + reinterpret_cast(tc.span_id().data()), + tc.span_id().size(), + tc.has_trace_flags() ? static_cast(tc.trace_flags()) + : std::uint8_t{0}); + } + } +#endif + // No propagated context — start a fresh trace root (not an ambient child). + return SpanGuard::freshRoot( + TraceCategory::Consensus, seg::consensus, consensus::span::op::validationReceive); +} + +} // namespace xrpl::telemetry diff --git a/src/xrpld/telemetry/MetricMacros.h b/src/xrpld/telemetry/MetricMacros.h new file mode 100644 index 00000000000..6bea92d8e3b --- /dev/null +++ b/src/xrpld/telemetry/MetricMacros.h @@ -0,0 +1,515 @@ +#pragma once + +// cspell:ignore ISTOGRAM +// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's +// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. + +/** + * Call-site OTel metric macros. + * + * Adds a new OTel metric instrument entirely at the call site -- no member + * field, no init line, no wrapper method in MetricsRegistry. Covers every + * instrument kind the OTel Metrics API defines: + * + * Synchronous (create-once via std::call_once, then record on every call): + * Counter XRPL_METRIC_COUNTER_INC / _ADD [+ _LABELED] + * UpDownCounter XRPL_METRIC_UPDOWN_ADD [+ _LABELED] + * Histogram XRPL_METRIC_HISTOGRAM_RECORD [+ _LABELED] + * Gauge XRPL_METRIC_GAUGE_RECORD [+ _LABELED] + * (requires OPENTELEMETRY_ABI_VERSION_NO >= 2; + * this repo currently builds ABI v1 -- see the + * static_assert branch below) + * + * Asynchronous/observable (register a callback ONCE, eagerly, during + * construction/init -- see the "Observable" note below): + * Observable Counter XRPL_METRIC_OBSERVABLE_COUNTER_REGISTER + * Observable UpDownCounter XRPL_METRIC_OBSERVABLE_UPDOWN_REGISTER + * Observable Gauge XRPL_METRIC_OBSERVABLE_GAUGE_REGISTER + * + * When XRPL_ENABLE_TELEMETRY is not defined, every macro expands to a + * no-op statement, so call sites never need their own #ifdef. + * + * Example usage -- plain counter: + * @code + * void RCLConsensus::Adaptor::doAccept() + * { + * // ... existing consensus-accept logic ... + * XRPL_METRIC_COUNTER_INC(app_, "ledgers_closed_total", + * "Total ledgers closed by consensus"); + * } + * @endcode + * + * Example usage -- labeled counter (edge case: per-reason tally): + * @code + * void TxQ::apply(...) + * { + * if (queueIsFull) + * XRPL_METRIC_COUNTER_INC_LABELED(app, "txq_dropped_total", + * "Transactions refused admission to the queue", + * {{"reason", std::string("queue_full")}}); + * } + * @endcode + * + * Pass the label set as a bare brace-enclosed list, as above. Do not wrap + * it in an extra pair of parentheses: the list is forwarded verbatim into + * the OTel `Add()`/`Record()` call, which takes an initializer_list, and + * the extra parentheses do not compile. + * + * Wrap each label *value* in `std::string`. `AttributeValue` is a variant + * in which a bare `const char*` selects the boolean alternative, so an + * unwrapped literal is recorded as `true`. + * + * Example usage -- UpDownCounter (edge case: value that can decrease): + * @code + * void ServerHandler::onRpcStart() + * { + * XRPL_METRIC_UPDOWN_ADD(app_, "rpc_in_flight_requests", + * "RPC requests currently executing", 1); + * } + * void ServerHandler::onRpcFinish() + * { + * XRPL_METRIC_UPDOWN_ADD(app_, "rpc_in_flight_requests", + * "RPC requests currently executing", -1); + * } + * @endcode + * + * Example usage -- observable gauge registered from a non-MetricsRegistry + * class (edge case: a subsystem exposing its own live state): + * @code + * SomeSubsystem::SomeSubsystem(ServiceRegistry& app) : app_(app) + * { + * XRPL_METRIC_OBSERVABLE_GAUGE_REGISTER( + * app_, "some_subsystem_queue_depth", "Current queue depth", + * [this] { return static_cast(queue_.size()); }); + * } + * @endcode + * + * @note A histogram whose values can exceed ~10,000 units (e.g. a + * microsecond duration beyond 10ms) needs an explicit-bucket View, which + * OTel can only register at MeterProvider construction time -- this + * cannot be done from a call site. See Limitation 2. Register such a + * view in MetricsRegistry::initExporterAndProvider() as today; the + * histogram-record call itself can still use the macro. + * + * @note Only call the SYNCHRONOUS macros (Counter/UpDownCounter/ + * Histogram/Gauge) from code that runs AFTER MetricsRegistry::start() has + * completed (RPC handlers, job callbacks, consensus rounds, tx apply, peer + * message handlers). See Limitation 1. + * + * @note The OBSERVABLE registration macros are the opposite: call them + * EAGERLY, exactly once, from constructor/init code -- never from a hot + * path. Repeated calls at the same call site register a NEW callback + * each time (no create-once caching, unlike the synchronous macros), + * which leaks callbacks. See Limitation 3. + * + * @note There is no way to read back a synchronous instrument's current + * accumulated value from application code -- the OTel API is + * write-only/push-based by design. If your logic needs both to record a + * metric AND read its running value, keep your own state (std::atomic or + * similar) and separately feed OTel via these macros. See "Use Case 4" in + * tasks/metric-macro-plan.md. + */ + +// On Windows, OTel's spin_lock_mutex.h (transitively included from +// MetricsRegistry.h) defines _WINSOCKAPI_ and includes , which +// pulls in WinSock 1. ServiceRegistry.h below then includes , +// whose socket_types.hpp requires winsock2.h first and errors out if WinSock 1 +// arrived earlier. Pre-including boost's socket types header here gets +// winsock2.h in before the OTel headers, so any translation unit that includes +// MetricMacros.h first (e.g. the telemetry unit tests) still compiles. The +// production MetricsRegistry.cpp carries the same guard. +#ifdef _MSC_VER +#include +#endif + +#include // IWYU pragma: keep + +#include // IWYU pragma: keep + +#ifdef XRPL_ENABLE_TELEMETRY + +#include // IWYU pragma: keep +#include // IWYU pragma: keep + +#define XRPL_METRIC_COUNTER_INC(app, name, description) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + static opentelemetry::nostd::unique_ptr> \ + xrpl_counter_; \ + static std::once_flag xrpl_once_; \ + std::call_once(xrpl_once_, [&] { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + xrpl_counter_ = xrpl_m_->CreateUInt64Counter((name), (description)); \ + }); \ + if (xrpl_counter_) \ + xrpl_counter_->Add(1); \ + } \ + } while (false) + +// The label set is passed as trailing variadic arguments so a +// brace-enclosed initializer list (e.g. {{"reason", std::string("x")}}), +// which contains a top-level comma, survives preprocessing as a single +// logical argument. __VA_ARGS__ re-joins it verbatim into the Add() call. +#define XRPL_METRIC_COUNTER_INC_LABELED(app, name, description, ...) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + static opentelemetry::nostd::unique_ptr> \ + xrpl_counter_; \ + static std::once_flag xrpl_once_; \ + std::call_once(xrpl_once_, [&] { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + xrpl_counter_ = xrpl_m_->CreateUInt64Counter((name), (description)); \ + }); \ + if (xrpl_counter_) \ + xrpl_counter_->Add(1, __VA_ARGS__); \ + } \ + } while (false) + +// Same as XRPL_METRIC_COUNTER_INC, but increments by a caller-supplied amount +// instead of a fixed 1 (e.g. bytes transferred, batch sizes). +#define XRPL_METRIC_COUNTER_ADD(app, name, description, amount) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + static opentelemetry::nostd::unique_ptr> \ + xrpl_counter_; \ + static std::once_flag xrpl_once_; \ + std::call_once(xrpl_once_, [&] { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + xrpl_counter_ = xrpl_m_->CreateUInt64Counter((name), (description)); \ + }); \ + if (xrpl_counter_) \ + xrpl_counter_->Add(amount); \ + } \ + } while (false) + +// amount is fixed; the trailing variadic args carry the label set (see the +// note on XRPL_METRIC_COUNTER_INC_LABELED for why labels are variadic). +#define XRPL_METRIC_COUNTER_ADD_LABELED(app, name, description, amount, ...) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + static opentelemetry::nostd::unique_ptr> \ + xrpl_counter_; \ + static std::once_flag xrpl_once_; \ + std::call_once(xrpl_once_, [&] { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + xrpl_counter_ = xrpl_m_->CreateUInt64Counter((name), (description)); \ + }); \ + if (xrpl_counter_) \ + xrpl_counter_->Add(amount, __VA_ARGS__); \ + } \ + } while (false) + +// UpDownCounter: like COUNTER_ADD, but the underlying instrument permits a +// negative amount (Use Case 2 -- e.g. in-flight request count, +1 on start +// / -1 on finish from two different points in the same or different call +// sites). A plain Counter's Add() must never see a negative value per the +// OTel API contract; use this macro, not COUNTER_ADD, whenever the value +// can decrease. +#define XRPL_METRIC_UPDOWN_ADD(app, name, description, amount) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + static opentelemetry::nostd::unique_ptr< \ + opentelemetry::metrics::UpDownCounter> \ + xrpl_updown_; \ + static std::once_flag xrpl_once_; \ + std::call_once(xrpl_once_, [&] { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + xrpl_updown_ = xrpl_m_->CreateInt64UpDownCounter((name), (description)); \ + }); \ + if (xrpl_updown_) \ + xrpl_updown_->Add(amount); \ + } \ + } while (false) + +// amount may be negative; the trailing variadic args carry the label set +// (see the note on XRPL_METRIC_COUNTER_INC_LABELED for why labels are variadic). +#define XRPL_METRIC_UPDOWN_ADD_LABELED(app, name, description, amount, ...) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + static opentelemetry::nostd::unique_ptr< \ + opentelemetry::metrics::UpDownCounter> \ + xrpl_updown_; \ + static std::once_flag xrpl_once_; \ + std::call_once(xrpl_once_, [&] { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + xrpl_updown_ = xrpl_m_->CreateInt64UpDownCounter((name), (description)); \ + }); \ + if (xrpl_updown_) \ + xrpl_updown_->Add(amount, __VA_ARGS__); \ + } \ + } while (false) + +#define XRPL_METRIC_HISTOGRAM_RECORD(app, name, description, value) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + static opentelemetry::nostd::unique_ptr> \ + xrpl_hist_; \ + static std::once_flag xrpl_once_; \ + std::call_once(xrpl_once_, [&] { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + xrpl_hist_ = xrpl_m_->CreateDoubleHistogram((name), (description)); \ + }); \ + if (xrpl_hist_) \ + xrpl_hist_->Record(static_cast(value), opentelemetry::context::Context{}); \ + } \ + } while (false) + +// value is fixed; the trailing variadic args carry the label set (see the +// note on XRPL_METRIC_COUNTER_INC_LABELED for why labels are variadic). +#define XRPL_METRIC_HISTOGRAM_RECORD_LABELED(app, name, description, value, ...) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + static opentelemetry::nostd::unique_ptr> \ + xrpl_hist_; \ + static std::once_flag xrpl_once_; \ + std::call_once(xrpl_once_, [&] { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + xrpl_hist_ = xrpl_m_->CreateDoubleHistogram((name), (description)); \ + }); \ + if (xrpl_hist_) \ + xrpl_hist_->Record( \ + static_cast(value), __VA_ARGS__, opentelemetry::context::Context{}); \ + } \ + } while (false) + +// Synchronous Gauge: last-value snapshot, not a distribution (contrast +// Histogram) and not a running total (contrast Counter/UpDownCounter). +// ABI-gated: opentelemetry-cpp only exposes CreateInt64Gauge/ +// CreateDoubleGauge when OPENTELEMETRY_ABI_VERSION_NO >= 2. This project's +// Conan build currently pins ABI v1 (verified: +// .build/build/generators/opentelemetry-cpp-release-x86_64-data.cmake sets +// OPENTELEMETRY_ABI_VERSION_NO=1), so the real path below is presently +// dead code on this codebase's build -- shipped anyway so it activates +// automatically the day the ABI version is bumped, and so a developer who +// reaches for "just the current value, not a distribution" sees an +// actionable compile error now instead of silently reaching for the wrong +// instrument kind (misusing Histogram or UpDownCounter as a gauge +// substitute is explicitly discouraged -- see Design/taxonomy section). +#if OPENTELEMETRY_ABI_VERSION_NO >= 2 +#define XRPL_METRIC_GAUGE_RECORD(app, name, description, value) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + static opentelemetry::nostd::unique_ptr> \ + xrpl_gauge_; \ + static std::once_flag xrpl_once_; \ + std::call_once(xrpl_once_, [&] { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + xrpl_gauge_ = xrpl_m_->CreateDoubleGauge((name), (description)); \ + }); \ + if (xrpl_gauge_) \ + xrpl_gauge_->Record( \ + static_cast(value), opentelemetry::context::Context{}); \ + } \ + } while (false) +// value is fixed; the trailing variadic args carry the label set (see the +// note on XRPL_METRIC_COUNTER_INC_LABELED for why labels are variadic). +#define XRPL_METRIC_GAUGE_RECORD_LABELED(app, name, description, value, ...) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + static opentelemetry::nostd::unique_ptr> \ + xrpl_gauge_; \ + static std::once_flag xrpl_once_; \ + std::call_once(xrpl_once_, [&] { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + xrpl_gauge_ = xrpl_m_->CreateDoubleGauge((name), (description)); \ + }); \ + if (xrpl_gauge_) \ + xrpl_gauge_->Record( \ + static_cast(value), __VA_ARGS__, opentelemetry::context::Context{}); \ + } \ + } while (false) +#else +#define XRPL_METRIC_GAUGE_RECORD(app, name, description, value) \ + static_assert( \ + false, \ + "XRPL_METRIC_GAUGE_RECORD requires OPENTELEMETRY_ABI_VERSION_NO >= 2 (this build " \ + "uses ABI v1). Use XRPL_METRIC_OBSERVABLE_GAUGE_REGISTER with your own state, or " \ + "bump OPENTELEMETRY_ABI_VERSION_NO as a separate, reviewed change.") +#define XRPL_METRIC_GAUGE_RECORD_LABELED(app, name, description, value, ...) \ + static_assert( \ + false, \ + "XRPL_METRIC_GAUGE_RECORD_LABELED requires OPENTELEMETRY_ABI_VERSION_NO >= 2 " \ + "(this build uses ABI v1). Use XRPL_METRIC_OBSERVABLE_GAUGE_REGISTER with your " \ + "own state, or bump OPENTELEMETRY_ABI_VERSION_NO as a separate, reviewed change.") +#endif // OPENTELEMETRY_ABI_VERSION_NO >= 2 + +// ----------------------------------------------------------------- +// Observable/async instrument registration (Use Case 5). Unlike the +// synchronous macros above, these do NOT lazily create-on-first-call -- +// they register a callback with the SDK immediately, at the call site, +// the moment the macro executes. Callers MUST invoke this during +// construction/init, before the server is fully live (same timing rule +// MetricsRegistry::registerAsyncGauges() already follows for its own +// gauges -- see Limitation 3). Calling it from a hot-path function +// instead of an init path re-registers a new callback on every call, +// which leaks callbacks and is NOT what this macro is for. +// +// The callable is captured in a heap-allocated std::function, and its +// address is passed as the `void* state` to AddCallback (whose signature, +// `void (*)(ObserverResult, void*)`, is a raw C function pointer -- it +// cannot bind a capturing lambda directly). A static trampoline +// function reinterprets `state` back to the std::function and invokes +// it inside the callback. The heap allocation is intentionally leaked +// for the process lifetime (matches every existing ObservableGauge in +// MetricsRegistry, which are member fields with the same lifetime as +// the registry itself) -- do not "fix" this with a smart pointer that +// frees before the reader thread's last collection tick. +// ----------------------------------------------------------------- +#define XRPL_METRIC_OBSERVABLE_GAUGE_REGISTER(app, name, description, valueFn) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + { \ + auto* xrpl_fn_ = new std::function(valueFn); \ + auto xrpl_inst_ = xrpl_m_->CreateInt64ObservableGauge((name), (description)); \ + xrpl_inst_->AddCallback( \ + [](opentelemetry::metrics::ObserverResult result, void* state) { \ + auto* fn = static_cast*>(state); \ + try \ + { \ + opentelemetry::nostd::get>>(result) \ + ->Observe((*fn)()); \ + } \ + catch (...) \ + { \ + } \ + }, \ + xrpl_fn_); \ + } \ + } \ + } while (false) + +#define XRPL_METRIC_OBSERVABLE_COUNTER_REGISTER(app, name, description, valueFn) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + { \ + auto* xrpl_fn_ = new std::function(valueFn); \ + auto xrpl_inst_ = xrpl_m_->CreateInt64ObservableCounter((name), (description)); \ + xrpl_inst_->AddCallback( \ + [](opentelemetry::metrics::ObserverResult result, void* state) { \ + auto* fn = static_cast*>(state); \ + try \ + { \ + opentelemetry::nostd::get>>(result) \ + ->Observe((*fn)()); \ + } \ + catch (...) \ + { \ + } \ + }, \ + xrpl_fn_); \ + } \ + } \ + } while (false) + +#define XRPL_METRIC_OBSERVABLE_UPDOWN_REGISTER(app, name, description, valueFn) \ + do \ + { \ + if (auto* xrpl_mr_ = (app).getMetricsRegistry(); xrpl_mr_ && xrpl_mr_->isEnabled()) \ + { \ + if (auto xrpl_m_ = xrpl_mr_->meter()) \ + { \ + auto* xrpl_fn_ = new std::function(valueFn); \ + auto xrpl_inst_ = \ + xrpl_m_->CreateInt64ObservableUpDownCounter((name), (description)); \ + xrpl_inst_->AddCallback( \ + [](opentelemetry::metrics::ObserverResult result, void* state) { \ + auto* fn = static_cast*>(state); \ + try \ + { \ + opentelemetry::nostd::get>>(result) \ + ->Observe((*fn)()); \ + } \ + catch (...) \ + { \ + } \ + }, \ + xrpl_fn_); \ + } \ + } \ + } while (false) + +#else // !XRPL_ENABLE_TELEMETRY + +#define XRPL_METRIC_COUNTER_INC(app, name, description) \ + do \ + { \ + } while (false) +#define XRPL_METRIC_COUNTER_INC_LABELED(app, name, description, ...) \ + do \ + { \ + } while (false) +#define XRPL_METRIC_COUNTER_ADD(app, name, description, amount) \ + do \ + { \ + } while (false) +#define XRPL_METRIC_COUNTER_ADD_LABELED(app, name, description, amount, ...) \ + do \ + { \ + } while (false) +#define XRPL_METRIC_UPDOWN_ADD(app, name, description, amount) \ + do \ + { \ + } while (false) +#define XRPL_METRIC_UPDOWN_ADD_LABELED(app, name, description, amount, ...) \ + do \ + { \ + } while (false) +#define XRPL_METRIC_HISTOGRAM_RECORD(app, name, description, value) \ + do \ + { \ + } while (false) +#define XRPL_METRIC_HISTOGRAM_RECORD_LABELED(app, name, description, value, ...) \ + do \ + { \ + } while (false) +#define XRPL_METRIC_GAUGE_RECORD(app, name, description, value) \ + do \ + { \ + } while (false) +#define XRPL_METRIC_GAUGE_RECORD_LABELED(app, name, description, value, ...) \ + do \ + { \ + } while (false) +#define XRPL_METRIC_OBSERVABLE_GAUGE_REGISTER(app, name, description, valueFn) \ + do \ + { \ + } while (false) +#define XRPL_METRIC_OBSERVABLE_COUNTER_REGISTER(app, name, description, valueFn) \ + do \ + { \ + } while (false) +#define XRPL_METRIC_OBSERVABLE_UPDOWN_REGISTER(app, name, description, valueFn) \ + do \ + { \ + } while (false) + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp new file mode 100644 index 00000000000..384a71e6056 --- /dev/null +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -0,0 +1,1599 @@ +/** + * MetricsRegistry implementation — OpenTelemetry metric instruments for xrpld. + * + * This file contains: + * - Construction / destruction logic for the OTel MeterProvider pipeline. + * - Synchronous instrument creation (counters, histograms) for RPC, job + * queue, and NodeStore I/O metrics. + * - Observable gauge callback registration for cache hit rates, TxQ state, + * CountedObject instances, load factors, and NodeStore queue depth. + * - No-op stubs when XRPL_ENABLE_TELEMETRY is not defined. + */ + +// On Windows, OTel's spin_lock_mutex.h (transitively included from +// MetricsRegistry.h) defines _WINSOCKAPI_ and includes . +// This poisons the include state for boost/asio/detail/socket_types.hpp, +// which requires winsock2.h to be included first. Pre-including the +// boost/asio socket types header gets winsock2.h in before the OTel +// headers can interfere. +#ifdef _MSC_VER +#include +#endif + +#include + +#ifdef XRPL_ENABLE_TELEMETRY + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace metric_sdk = opentelemetry::sdk::metrics; +namespace otlp_http = opentelemetry::exporter::otlp; +namespace resource = opentelemetry::sdk::resource; + +namespace { + +// Microsecond-valued duration histogram instrument names. Each is +// referenced twice — once to register the explicit-bucket view and once +// to create the instrument — so they are named constants to keep the two +// sites in sync (a mismatch would silently drop the bucket override). +constexpr char kJobQueuedDurationUs[] = "job_queued_us"; +constexpr char kJobRunningDurationUs[] = "job_running_us"; +constexpr char kRpcMethodDurationUs[] = "rpc_method_us"; + +// Attribute (label) keys for the job instruments. Each is referenced from +// several record sites, and a counter and its histogram must carry exactly +// the same key spelling or the two series cannot be joined in a query. +constexpr char kJobTypeLabel[] = "job_type"; +constexpr char kHandlerLabel[] = "handler"; + +/** + * Bucket boundaries for microsecond-valued duration instruments. + * + * 100 µs, 500 µs, 1 ms, 5 ms, 10 ms, 25 ms, 50 ms, 100 ms, 250 ms, 500 ms, + * 1 s, 2.5 s, 5 s, 10 s, 30 s, 60 s. Covers sub-millisecond jobs through + * multi-second stalls without saturating. + */ +constexpr std::array kMicrosecondBoundaries{ + 100.0, + 500.0, + 1'000.0, + 5'000.0, + 10'000.0, + 25'000.0, + 50'000.0, + 100'000.0, + 250'000.0, + 500'000.0, + 1'000'000.0, + 2'500'000.0, + 5'000'000.0, + 10'000'000.0, + 30'000'000.0, + 60'000'000.0}; + +/** + * Register an explicit-bucket histogram view. + * + * The SDK's default boundaries top out at 10,000, so any instrument whose + * values exceed that saturates and every quantile reads as the ceiling. + * + * @param views The registry to add the view to. + * @param name Instrument name to match (e.g. "job_running_us"). + * @param boundaries Bucket upper bounds, ascending. + */ +void +addHistogramView( + metric_sdk::ViewRegistry& views, + std::string const& name, + std::vector boundaries) +{ + auto config = std::make_shared(); + config->boundaries_ = std::move(boundaries); + + auto selector = metric_sdk::InstrumentSelectorFactory::Create( + metric_sdk::InstrumentType::kHistogram, name, ""); + auto meterSelector = metric_sdk::MeterSelectorFactory::Create("xrpld", "1.0.0", ""); + auto view = + metric_sdk::ViewFactory::Create(name, "", metric_sdk::AggregationType::kHistogram, config); + + views.AddView(std::move(selector), std::move(meterSelector), std::move(view)); +} + +/** + * Register the microsecond-ladder view for a duration instrument. + * + * Job wait/run times and RPC latencies routinely exceed the SDK default + * ceiling, so they all share `kMicrosecondBoundaries`. + * + * @param views The registry to add the view to. + * @param name Instrument name to match. + */ +void +addMicrosecondHistogramView(metric_sdk::ViewRegistry& views, std::string const& name) +{ + addHistogramView(views, name, {kMicrosecondBoundaries.begin(), kMicrosecondBoundaries.end()}); +} + +} // namespace + +#endif // XRPL_ENABLE_TELEMETRY + +namespace xrpl::telemetry { + +MetricsRegistry::MetricsRegistry( + [[maybe_unused]] bool enabled, + [[maybe_unused]] ServiceRegistry& app, + [[maybe_unused]] beast::Journal journal) + : enabled_(enabled) +#ifdef XRPL_ENABLE_TELEMETRY + , app_(app) + , journal_(journal) +#endif +{ +} + +MetricsRegistry::~MetricsRegistry() +{ + stop(); +} + +void +MetricsRegistry::start(std::string const& endpoint, std::string const& instanceId) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_) + return; + + JLOG(journal_.info()) << "MetricsRegistry: starting, endpoint=" << endpoint + << ", instanceId=" << instanceId; + + initExporterAndProvider(endpoint, instanceId); + initSyncInstruments(); + registerAsyncGauges(); + + JLOG(journal_.info()) << "MetricsRegistry: started successfully"; +#else + (void)endpoint; + (void)instanceId; + (void)enabled_; +#endif // XRPL_ENABLE_TELEMETRY +} + +#ifdef XRPL_ENABLE_TELEMETRY +void +MetricsRegistry::initExporterAndProvider(std::string const& endpoint, std::string const& instanceId) +{ + // Configure OTLP/HTTP metric exporter. + otlp_http::OtlpHttpMetricExporterOptions exporterOpts; + exporterOpts.url = endpoint; + auto exporter = otlp_http::OtlpHttpMetricExporterFactory::Create(exporterOpts); + + // Configure periodic reader with 10-second export interval. + metric_sdk::PeriodicExportingMetricReaderOptions readerOpts; + readerOpts.export_interval_millis = std::chrono::milliseconds(10000); + readerOpts.export_timeout_millis = std::chrono::milliseconds(5000); + auto reader = + metric_sdk::PeriodicExportingMetricReaderFactory::Create(std::move(exporter), readerOpts); + + // Configure resource attributes so Prometheus service_instance_id labels + // distinguish metrics from different nodes (matches OTelCollector setup). + resource::ResourceAttributes attrs; + // Use std::string, not a string literal: ResourceAttributes stores an + // OTel AttributeValue variant whose char-const* overload binds to bool, + // so "xrpld" would be recorded as the boolean true. std::string selects + // the string alternative and the value round-trips as service.name=xrpld. + attrs[opentelemetry::semconv::service::kServiceName] = std::string("xrpld"); + if (!instanceId.empty()) + attrs[opentelemetry::semconv::service::kServiceInstanceId] = instanceId; + auto resourceAttrs = resource::Resource::Create(attrs); + + // Build a view registry with explicit microsecond buckets for the + // duration histograms. Without this they use the SDK default buckets + // (max 10,000 = 10 ms), saturating every quantile at 10 ms. + auto views = std::make_unique(); + addMicrosecondHistogramView(*views, kJobQueuedDurationUs); + addMicrosecondHistogramView(*views, kJobRunningDurationUs); + addMicrosecondHistogramView(*views, kRpcMethodDurationUs); + // Recorded at its PeerImp.cpp call site, not created here, so the name + // comes from the shared constant both sites use. + addMicrosecondHistogramView(*views, kGetObjectLookupUs); + + // The remaining two GetObject histograms are not durations, so the + // microsecond ladder above does not fit them. Both still need explicit + // boundaries: the SDK default stops at 10,000 and both ranges exceed it. + // + // Object counts run 1..kHardMaxReplyNodes (12288). The honest sync path + // asks for at most 8, so the low buckets are fine-grained and the upper + // ones follow the charge size bands (64, 1024) up to the hard cap. + addHistogramView( + *views, + kGetObjectRequestObjects, + {1.0, 2.0, 4.0, 8.0, 16.0, 64.0, 256.0, 1'024.0, 4'096.0, 12'288.0}); + + // Charge values span 0 (free tier) to ~99k for a full-size all-miss + // request. Boundaries bracket the resource thresholds that decide a + // peer's fate -- kWarningThreshold (5000) and kDropThreshold (25000) -- + // so a dashboard can show how close charges run to each. + addHistogramView( + *views, + kGetObjectCharge, + {0.0, 100.0, 500.0, 1'000.0, 5'000.0, 10'000.0, 25'000.0, 50'000.0, 100'000.0}); + + // Create MeterProvider with resource, then attach the metric reader. + provider_ = metric_sdk::MeterProviderFactory::Create(std::move(views), resourceAttrs); + provider_->AddMetricReader(std::move(reader)); + + // Get a meter for all xrpld instruments. + meter_ = provider_->GetMeter("xrpld", "1.0.0"); +} + +void +MetricsRegistry::initSyncInstruments() +{ + // RPC per-method counters and histogram. + rpcStartedCounter_ = + meter_->CreateUInt64Counter("rpc_method_started_total", "Total RPC method calls started"); + rpcFinishedCounter_ = meter_->CreateUInt64Counter( + "rpc_method_finished_total", "Total RPC method calls completed successfully"); + rpcErroredCounter_ = meter_->CreateUInt64Counter( + "rpc_method_errored_total", "Total RPC method calls that errored"); + rpcDurationHistogram_ = meter_->CreateDoubleHistogram( + kRpcMethodDurationUs, "RPC method execution time in microseconds"); + + // Job queue per-type counters and histograms. + jobQueuedCounter_ = meter_->CreateUInt64Counter("job_queued_total", "Total jobs enqueued"); + jobStartedCounter_ = meter_->CreateUInt64Counter("job_started_total", "Total jobs started"); + jobFinishedCounter_ = meter_->CreateUInt64Counter("job_finished_total", "Total jobs completed"); + jobQueuedDurationHistogram_ = meter_->CreateDoubleHistogram( + kJobQueuedDurationUs, "Time jobs spent waiting in the queue (microseconds)"); + jobRunningDurationHistogram_ = + meter_->CreateDoubleHistogram(kJobRunningDurationUs, "Job execution time in microseconds"); + + // --- External dashboard parity counters (Task 7.14) --- + ledgersClosedCounter_ = + meter_->CreateUInt64Counter("ledgers_closed_total", "Total ledgers closed by consensus"); + validationsSentCounter_ = meter_->CreateUInt64Counter( + "validations_sent_total", "Total validations sent by this node"); + validationsCheckedCounter_ = meter_->CreateUInt64Counter( + "validations_checked_total", "Total network validations received and checked"); + stateChangesCounter_ = + meter_->CreateUInt64Counter("state_changes_total", "Total operating mode changes"); + // jq_trans_overflow_total is observed from Overlay's existing cumulative + // atomic (Overlay::getJqTransOverflow()) rather than pushed. The overlay + // owns the only increment site (PeerImp), so an ObservableCounter reads the + // live total each collection cycle without threading a push path through + // develop-owned overlay code. + jqTransOverflowObservable_ = meter_->CreateInt64ObservableCounter( + "jq_trans_overflow_total", "Total job queue transaction overflows"); + jqTransOverflowObservable_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + try + { + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(self->app_.getOverlay().getJqTransOverflow())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); + ledgerHistoryMismatchCounter_ = meter_->CreateUInt64Counter( + "ledger_history_mismatch_total", "Total built-vs-validated ledger mismatches by reason"); + txqExpiredCounter_ = meter_->CreateUInt64Counter( + "txq_expired_total", "Total transactions expired out of the transaction queue"); + txqDroppedCounter_ = meter_->CreateUInt64Counter( + "txq_dropped_total", "Total transactions refused admission to the queue by reason"); + // Note: validation_agreements_total / validation_missed_total are monotonic + // ObservableCounters created in registerValidationTotalsCounters() (below). +} +#endif // XRPL_ENABLE_TELEMETRY + +void +MetricsRegistry::detachCallbacks() noexcept +{ +#ifdef XRPL_ENABLE_TELEMETRY + // Release so every subsequent callback acquire-load sees true. + callbacksDetached_.store(true, std::memory_order_release); +#endif // XRPL_ENABLE_TELEMETRY +} + +void +MetricsRegistry::stop() +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!provider_) + return; + + JLOG(journal_.info()) << "MetricsRegistry: stopping"; + + // Belt-and-suspenders: detachCallbacks() should have already been + // called by Application shutdown before any service the callbacks + // observe was stopped. Setting the flag here is redundant for a + // correct caller but protects against a future caller that forgets + // to detach first. + callbacksDetached_.store(true, std::memory_order_release); + + // SDK teardown order: Shutdown() stops the PeriodicExportingMetricReader + // thread (so no further gauge callbacks fire) and performs the final + // collect-and-export drain itself. The trailing ForceFlush() is a + // redundant safety net (a no-op once the reader is shut down), then + // reset() destroys the provider. + provider_->Shutdown(); + provider_->ForceFlush(); + provider_.reset(); + + JLOG(journal_.info()) << "MetricsRegistry: stopped"; +#endif // XRPL_ENABLE_TELEMETRY +} + +// ----------------------------------------------------------------- +// Synchronous instrument recording — RPC metrics (Task 9.4) +// ----------------------------------------------------------------- + +void +MetricsRegistry::recordRpcStarted(std::string_view method) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_ || !rpcStartedCounter_) + return; + rpcStartedCounter_->Add(1, {{"method", std::string(method)}}); +#else + (void)method; + (void)enabled_; +#endif +} + +void +MetricsRegistry::recordRpcFinished(std::string_view method, std::int64_t durationUs) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_ || !rpcFinishedCounter_) + return; + rpcFinishedCounter_->Add(1, {{"method", std::string(method)}}); + if (rpcDurationHistogram_) + { + rpcDurationHistogram_->Record( + static_cast(durationUs), + {{"method", std::string(method)}}, + opentelemetry::context::Context{}); + } +#else + (void)method; + (void)durationUs; + (void)enabled_; +#endif +} + +void +MetricsRegistry::recordRpcErrored(std::string_view method, std::int64_t durationUs) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_ || !rpcErroredCounter_) + return; + rpcErroredCounter_->Add(1, {{"method", std::string(method)}}); + if (rpcDurationHistogram_) + { + rpcDurationHistogram_->Record( + static_cast(durationUs), + {{"method", std::string(method)}}, + opentelemetry::context::Context{}); + } +#else + (void)method; + (void)durationUs; + (void)enabled_; +#endif +} + +// ----------------------------------------------------------------- +// Synchronous instrument recording — Job Queue metrics (Task 9.5) +// ----------------------------------------------------------------- + +void +MetricsRegistry::recordJobQueued(std::string_view jobType, std::string_view jobName) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_ || !jobQueuedCounter_) + return; + jobQueuedCounter_->Add( + 1, + {{kJobTypeLabel, std::string(jobType)}, + {kHandlerLabel, std::string(sanitiseHandler(jobName))}}); +#else + (void)jobType; + (void)jobName; + (void)enabled_; +#endif +} + +void +MetricsRegistry::recordJobStarted( + std::string_view jobType, + std::string_view jobName, + std::int64_t queuedDurUs) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_ || !jobStartedCounter_) + return; + // Build the attribute pair once: both the counter and the histogram + // must carry the identical label set or they cannot be joined. + std::string const handler(sanitiseHandler(jobName)); + jobStartedCounter_->Add(1, {{kJobTypeLabel, std::string(jobType)}, {kHandlerLabel, handler}}); + if (jobQueuedDurationHistogram_ && queuedDurUs >= 0) + { + // Guard against negative queued durations: the caller derives this + // from a steady-clock delta that can go slightly negative under clock + // skew or reordering. The OTel SDK rejects negative histogram values + // (logging a warning per call), so skip them rather than spam. + jobQueuedDurationHistogram_->Record( + static_cast(queuedDurUs), + {{kJobTypeLabel, std::string(jobType)}, {kHandlerLabel, handler}}, + opentelemetry::context::Context{}); + } +#else + (void)jobType; + (void)jobName; + (void)queuedDurUs; + (void)enabled_; +#endif +} + +void +MetricsRegistry::recordJobFinished( + std::string_view jobType, + std::string_view jobName, + std::int64_t runningDurUs) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (!enabled_ || !jobFinishedCounter_) + return; + std::string const handler(sanitiseHandler(jobName)); + jobFinishedCounter_->Add(1, {{kJobTypeLabel, std::string(jobType)}, {kHandlerLabel, handler}}); + if (jobRunningDurationHistogram_) + { + jobRunningDurationHistogram_->Record( + static_cast(runningDurUs), + {{kJobTypeLabel, std::string(jobType)}, {kHandlerLabel, handler}}, + opentelemetry::context::Context{}); + } +#else + (void)jobType; + (void)jobName; + (void)runningDurUs; + (void)enabled_; +#endif +} + +// ----------------------------------------------------------------- +// Observable gauge callbacks (Tasks 9.1, 9.2, 9.3, 9.6, 9.7) +// ----------------------------------------------------------------- + +#ifdef XRPL_ENABLE_TELEMETRY + +void +MetricsRegistry::registerAsyncGauges() +{ + // Each helper creates one observable instrument and attaches one + // callback. Keeping the registration bodies in separate methods + // preserves the 80-line-per-function limit enforced by CLAUDE.md. + registerCacheHitRateGauge(); + registerTxqGauge(); + registerObjectCountGauge(); + registerLoadFactorGauge(); + registerNodeStoreGauge(); + registerServerInfoGauge(); + registerBuildInfoGauge(); + registerCompleteLedgersGauge(); + registerDbMetricsGauge(); + registerValidatorHealthGauge(); + registerPeerQualityGauge(); + registerReduceRelayGauge(); + registerLedgerEconomyGauge(); + registerStateTrackingGauge(); + registerStorageDetailGauge(); + registerValidationAgreementGauge(); + registerValidationTotalsCounters(); +} + +void +MetricsRegistry::registerCacheHitRateGauge() +{ + // --- Task 9.2: Cache hit rate and size gauges --- + cacheHitRateGauge_ = + meter_->CreateDoubleObservableGauge("cache_metrics", "Cache hit rates and sizes"); + cacheHitRateGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + // SLE cache hit rate (0.0 - 1.0). + auto sleRate = app.getCachedSLEs().rate(); + opentelemetry::nostd::get>>(result) + ->Observe(sleRate, {{"metric", "SLE_hit_rate"}}); + + // Ledger cache hit rate. + // TaggedCache::getHitRate() returns 0-100; normalize to + // 0.0-1.0 so the Grafana panel using "percentunit" renders + // correctly. + auto ledgerRate = app.getLedgerMaster().getCacheHitRate() / 100.0; + opentelemetry::nostd::get>>(result) + ->Observe(ledgerRate, {{"metric", "ledger_hit_rate"}}); + + // AcceptedLedger cache hit rate (also 0-100 from + // TaggedCache; normalize to 0.0-1.0). + auto alRate = app.getAcceptedLedgerCache().getHitRate() / 100.0; + opentelemetry::nostd::get>>(result) + ->Observe(alRate, {{"metric", "AL_hit_rate"}}); + + // TreeNode cache size. + auto tnCacheSize = app.getNodeFamily().getTreeNodeCache()->getCacheSize(); + opentelemetry::nostd::get>>(result) + ->Observe( + static_cast(tnCacheSize), {{"metric", "treenode_cache_size"}}); + + // TreeNode track size. + auto tnTrackSize = app.getNodeFamily().getTreeNodeCache()->getTrackSize(); + opentelemetry::nostd::get>>(result) + ->Observe( + static_cast(tnTrackSize), {{"metric", "treenode_track_size"}}); + + // FullBelow cache size. + auto fbSize = app.getNodeFamily().getFullBelowCache()->size(); + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(fbSize), {{"metric", "fullbelow_size"}}); + + // AcceptedLedger cache size (entry count). + auto alSize = app.getAcceptedLedgerCache().size(); + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(alSize), {{"metric", "AL_size"}}); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerTxqGauge() +{ + // --- Task 9.3: TxQ metrics gauges --- + txqGauge_ = meter_->CreateDoubleObservableGauge("txq_metrics", "Transaction queue metrics"); + txqGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto const metrics = app.getTxQ().getMetrics(*app.getOpenLedger().current()); + + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + observe("txq_count", static_cast(metrics.txCount)); + observe( + "txq_max_size", + metrics.txQMaxSize ? static_cast(*metrics.txQMaxSize) : 0.0); + observe("txq_in_ledger", static_cast(metrics.txInLedger)); + observe("txq_per_ledger", static_cast(metrics.txPerLedger)); + observe( + "txq_reference_fee_level", + static_cast(metrics.referenceFeeLevel.fee())); + observe( + "txq_min_processing_fee_level", + static_cast(metrics.minProcessingFeeLevel.fee())); + observe("txq_med_fee_level", static_cast(metrics.medFeeLevel.fee())); + observe( + "txq_open_ledger_fee_level", + static_cast(metrics.openLedgerFeeLevel.fee())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if TxQ or OpenLedger are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerObjectCountGauge() +{ + // --- Task 9.6: Counted object instance gauges --- + objectCountGauge_ = meter_->CreateInt64ObservableGauge( + "object_count", "Live instance counts for key internal object types"); + objectCountGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + try + { + // Iterate through all CountedObject types via the linked + // list in CountedObjects. We report all types with count + // > 0, filtering to the key types of interest. + auto counts = CountedObjects::getInstance().getCounts(0); + for (auto const& [name, count] : counts) + { + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(count), {{"type", name}}); + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); +} + +void +MetricsRegistry::registerLoadFactorGauge() +{ + // --- Task 9.7: Load factor breakdown gauges --- + loadFactorGauge_ = + meter_->CreateDoubleObservableGauge("load_factor_metrics", "Fee load factor breakdown"); + loadFactorGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto& feeTrack = app.getFeeTrack(); + auto const loadBase = static_cast(feeTrack.getLoadBase()); + + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Combined load factor (server component). + observe( + "load_factor_server", static_cast(feeTrack.getLoadFactor()) / loadBase); + + // Individual factor components. + observe( + "load_factor_local", static_cast(feeTrack.getLocalFee()) / loadBase); + observe("load_factor_net", static_cast(feeTrack.getRemoteFee()) / loadBase); + observe( + "load_factor_cluster", + static_cast(feeTrack.getClusterFee()) / loadBase); + + // Fee escalation factors from TxQ. + auto const metrics = app.getTxQ().getMetrics(*app.getOpenLedger().current()); + auto refLevel = static_cast(metrics.referenceFeeLevel.fee()); + if (refLevel > 0) + { + observe( + "load_factor_fee_escalation", + static_cast(metrics.openLedgerFeeLevel.fee()) / refLevel); + observe( + "load_factor_fee_queue", + static_cast(metrics.minProcessingFeeLevel.fee()) / refLevel); + } + + // Combined load factor (max of server and fee escalation). + auto const loadFactorServer = feeTrack.getLoadFactor(); + auto const loadBaseServer = feeTrack.getLoadBase(); + double combined = static_cast(loadFactorServer) / loadBase; + if (refLevel > 0) + { + double const feeEscalation = + static_cast(metrics.openLedgerFeeLevel.fee()) * loadBaseServer / + refLevel; + if (feeEscalation > static_cast(loadFactorServer)) + { + combined = feeEscalation / loadBase; + } + } + observe("load_factor", combined); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerNodeStoreGauge() +{ + // --- Task 9.1: NodeStore I/O gauges --- + // The cumulative counters (reads, writes, bytes) are also exposed here + // as observable gauges. This avoids adding an xrpld dependency into the + // libxrpl nodestore code — the MetricsRegistry reads the existing atomic + // counters from Database via its public accessors. + nodeStoreGauge_ = meter_->CreateInt64ObservableGauge( + "nodestore_state", "NodeStore I/O counters, queue depth, and write load"); + nodeStoreGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto& db = app.getNodeStore(); + + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Cumulative counters (monotonically increasing). + observe("node_reads_total", static_cast(db.getFetchTotalCount())); + observe("node_reads_hit", static_cast(db.getFetchHitCount())); + observe("node_writes", static_cast(db.getStoreCount())); + observe("node_written_bytes", static_cast(db.getStoreSize())); + observe("node_read_bytes", static_cast(db.getFetchSize())); + + // Write load score (instantaneous). + observe("write_load", static_cast(db.getWriteLoad())); + + // Read queue depth (instantaneous). + json::Value obj(json::ValueType::Object); + db.getCountsJson(obj); + if (obj.isMember("read_queue")) + { + observe("read_queue", static_cast(obj["read_queue"].asUInt())); + } + + // Cumulative read duration (stored as JSON string, not int). + if (obj.isMember(jss::node_reads_duration_us)) + { + auto durStr = obj[jss::node_reads_duration_us].asString(); + if (!durStr.empty()) + { + observe("node_reads_duration_us", static_cast(std::stoll(durStr))); + } + } + + // Read thread pool stats (native JSON ints, no jss:: constants). + if (obj.isMember("read_request_bundle")) + { + observe( + "read_request_bundle", + static_cast(obj["read_request_bundle"].asInt())); + } + if (obj.isMember("read_threads_running")) + { + observe( + "read_threads_running", + static_cast(obj["read_threads_running"].asInt())); + } + if (obj.isMember("read_threads_total")) + { + observe( + "read_threads_total", + static_cast(obj["read_threads_total"].asInt())); + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); +} + +void +MetricsRegistry::registerServerInfoGauge() +{ + // --- Task 9.7a: Server info gauges --- + serverInfoGauge_ = + meter_->CreateInt64ObservableGauge("server_info", "Server-level health metrics"); + serverInfoGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Server operating mode (DISCONNECTED=0 .. FULL=4). + observe("server_state", static_cast(app.getOPs().getOperatingMode())); + + // Uptime in seconds since server start. + observe( + "uptime", static_cast(UptimeClock::now().time_since_epoch().count())); + + // Total peer count (inbound + outbound). + observe("peers", static_cast(app.getOverlay().size())); + + // Validated ledger sequence (0 if none yet). + observe( + "validated_ledger_seq", + static_cast(app.getLedgerMaster().getValidLedgerIndex())); + + // Current open ledger sequence. + observe( + "ledger_current_index", + static_cast(app.getLedgerMaster().getCurrentLedgerIndex())); + + // Cumulative resource-related peer disconnects. + observe( + "peer_disconnects_resources", + static_cast(app.getOverlay().getPeerDisconnectCharges())); + + // Last consensus round data (from JSON — only public API). + auto const consensusInfo = app.getOPs().getConsensusInfo(); + if (consensusInfo.isMember("previous_proposers")) + { + observe( + "last_close_proposers", + static_cast(consensusInfo["previous_proposers"].asUInt())); + } + if (consensusInfo.isMember("previous_mseconds")) + { + observe( + "last_close_converge_time_ms", + static_cast(consensusInfo["previous_mseconds"].asUInt())); + } + + // Network close time of the last closed ledger, as NetClock + // seconds since the XRPL epoch (2000-01-01). Unlike a span + // timestamp, a gauge value survives as a queryable time series, + // so dashboards can show last-close age (staleness) via + // now - value. The close interval comes from the + // ledgers_closed_total counter, not a delta of this gauge + // (a timestamp gauge's delta aliases to the scrape period). + // Skip until a ledger has closed. + if (auto const closed = app.getLedgerMaster().getClosedLedger()) + { + observe( + "last_close_time", + static_cast( + closed->header().closeTime.time_since_epoch().count())); + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerBuildInfoGauge() +{ + // --- Task 9.7b: Build info gauge --- + buildInfoGauge_ = meter_->CreateInt64ObservableGauge("build_info", "Build version information"); + buildInfoGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* /* state */) { + try + { + opentelemetry::nostd::get>>(result) + ->Observe(1, {{"version", std::string(BuildInfo::getVersionString())}}); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + } + }, + nullptr); +} + +void +MetricsRegistry::registerCompleteLedgersGauge() +{ + // --- Task 9.7c: Complete ledgers range gauge --- + completeLedgersGauge_ = meter_->CreateInt64ObservableGauge( + "complete_ledgers", "Complete ledger range start/end pairs"); + completeLedgersGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto const rangeStr = app.getLedgerMaster().getCompleteLedgers(); + if (rangeStr.empty() || rangeStr == "empty") + return; + + // Parse comma-separated ranges like + // "32570-50000,50005-75891421". + std::size_t rangeIndex = 0; + std::istringstream stream(rangeStr); + std::string segment; + while (std::getline(stream, segment, ',')) + { + auto const dashPos = segment.find('-'); + if (dashPos == std::string::npos || dashPos == 0 || + dashPos == segment.size() - 1) + continue; + + auto const startStr = segment.substr(0, dashPos); + auto const endStr = segment.substr(dashPos + 1); + + auto const idxStr = std::to_string(rangeIndex); + + opentelemetry::nostd::get>>(result) + ->Observe( + static_cast(std::stoll(startStr)), + {{"bound", "start"}, {"index", idxStr}}); + + opentelemetry::nostd::get>>(result) + ->Observe( + static_cast(std::stoll(endStr)), + {{"bound", "end"}, {"index", idxStr}}); + + ++rangeIndex; + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on parse error or if services not ready. + } + }, + this); +} + +void +MetricsRegistry::registerDbMetricsGauge() +{ + // --- Task 9.7d: Database size and fetch rate gauges --- + dbMetricsGauge_ = + meter_->CreateInt64ObservableGauge("db_metrics", "Database storage sizes and fetch rates"); + dbMetricsGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + auto& rdb = app.getRelationalDatabase(); + observe("db_kb_total", static_cast(rdb.getKBUsedAll())); + observe("db_kb_ledger", static_cast(rdb.getKBUsedLedger())); + observe("db_kb_transaction", static_cast(rdb.getKBUsedTransaction())); + + // Historical ledger fetches per minute. + observe( + "historical_perminute", + static_cast(app.getInboundLedgers().fetchRate())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerValidatorHealthGauge() +{ + // --- Task 7.9: Validator health gauges --- + validatorHealthGauge_ = + meter_->CreateDoubleObservableGauge("validator_health", "Validator health indicators"); + validatorHealthGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + observe("amendment_blocked", app.getOPs().isAmendmentBlocked() ? 1.0 : 0.0); + observe("unl_blocked", app.getOPs().isUNLBlocked() ? 1.0 : 0.0); + observe("validation_quorum", static_cast(app.getValidators().quorum())); + + // Days until UNL list expiry (-1 if no expiry known). + auto const expiry = app.getValidators().expires(); + if (expiry) + { + auto const now = app.getTimeKeeper().closeTime(); + auto const diffHours = + std::chrono::duration_cast(*expiry - now).count(); + observe("unl_expiry_days", static_cast(diffHours) / 24.0); + } + else + { + observe("unl_expiry_days", -1.0); + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerPeerQualityGauge() +{ + // --- Task 7.10: Peer quality gauges --- + // Uses Peer::json() to read latency and version since those accessors + // are not on the abstract Peer interface (they live on PeerImp). + peerQualityGauge_ = + meter_->CreateDoubleObservableGauge("peer_quality", "Peer network quality metrics"); + peerQualityGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Collect latencies, version info, and tracking state from + // each peer's JSON. + std::vector latencies; + int higherVersionCount = 0; + int totalPeers = 0; + int divergedCount = 0; + + // Encode a version string into BuildInfo's comparable numeric + // form. Peers report the full "xrpld-3.3.0-b0" string while + // our baseline is the bare "3.3.0-b0", and SemanticVersion + // requires a leading digit, so strip any non-digit prefix + // first. Numeric encoding avoids the lexicographic bug where + // "2.3.0" > "2.10.0" and "xrpld-..." > "3...". + auto const encodeVersion = [](std::string_view v) -> std::uint64_t { + auto const firstDigit = v.find_first_of("0123456789"); + if (firstDigit == std::string_view::npos) + return 0; + return BuildInfo::encodeSoftwareVersion(v.substr(firstDigit)); + }; + auto const ownEncoded = encodeVersion(BuildInfo::getVersionString()); + + app.getOverlay().foreach([&](std::shared_ptr const& peer) { + ++totalPeers; + auto const pj = peer->json(); + if (pj.isMember(jss::latency)) + { + latencies.push_back(pj[jss::latency].asInt()); + } + if (pj.isMember(jss::version)) + { + // Unparseable peer versions encode below ownEncoded, so + // the comparison correctly leaves them uncounted. + auto const pv = pj[jss::version].asString(); + if (encodeVersion(pv) > ownEncoded) + ++higherVersionCount; + } + // PeerImp::json() sets "track" to "diverged" when the peer's + // tracking state is Tracking::Diverged (i.e. it is following + // a different ledger chain than us). + if (pj.isMember(jss::track) && pj[jss::track].asString() == "diverged") + ++divergedCount; + }); + + // P90 latency across connected peers. + if (!latencies.empty()) + { + std::ranges::sort(latencies); + auto p90idx = static_cast(latencies.size() * 0.9); + if (p90idx >= latencies.size()) + p90idx = latencies.size() - 1; + observe("peer_latency_p90_ms", static_cast(latencies[p90idx])); + } + else + { + observe("peer_latency_p90_ms", 0.0); + } + + // Percentage of peers running a higher version. + double const higherPct = totalPeers > 0 + ? (static_cast(higherVersionCount) / totalPeers * 100.0) + : 0.0; + observe("peers_higher_version_pct", higherPct); + + // Count peers diverged from our ledger chain, read from the + // peer's "track" JSON field (set by PeerImp::json()). Diverged + // peers are following a different chain and are a leading + // indicator of local sync trouble. + observe("peers_insane_count", static_cast(divergedCount)); + + // Binary flag: recommend upgrade if >60% run a newer version. + observe("upgrade_recommended", higherPct > 60.0 ? 1.0 : 0.0); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerReduceRelayGauge() +{ + // Transaction reduce-relay efficiency. Overlay::txMetrics() exposes the + // rolling averages as a JSON object with string values (std::to_string), + // so parse each field. A high suppressed:selected ratio proves the + // feature is saving bandwidth; a high not_enabled count means stale peers + // force full relay. + reduceRelayGauge_ = meter_->CreateInt64ObservableGauge( + "reduce_relay_metrics", "Transaction reduce-relay efficiency metrics"); + reduceRelayGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto const tm = app.getOverlay().txMetrics(); + + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Each field is a decimal string; emit when present and parseable. + auto observeField = [&](auto const& field, char const* name) { + if (tm.isMember(field)) + { + auto const s = tm[field].asString(); + if (!s.empty()) + observe(name, static_cast(std::stoll(s))); + } + }; + + observeField(jss::txr_selected_cnt, "selected_peers"); + observeField(jss::txr_suppressed_cnt, "suppressed_peers"); + observeField(jss::txr_not_enabled_cnt, "not_enabled_peers"); + observeField(jss::txr_missing_tx_freq, "missing_tx_freq"); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready or a value is + // not parseable. + } + }, + this); +} + +void +MetricsRegistry::registerLedgerEconomyGauge() +{ + // --- Task 7.11: Ledger economy gauges --- + ledgerEconomyGauge_ = + meter_->CreateDoubleObservableGauge("ledger_economy", "Ledger fee and economy metrics"); + ledgerEconomyGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Local fee (drops). + observe("base_fee_xrp", static_cast(app.getFeeTrack().getLocalFee())); + + // Reserve values from the validated ledger. + auto const ledger = app.getLedgerMaster().getValidatedLedger(); + if (ledger) + { + auto const& fees = ledger->fees(); + // Base reserve = one account, zero owned objects: + // accountReserve(ownerCount=0, accountCount=1) == reserve. + observe( + "reserve_base_xrp", static_cast(fees.accountReserve(0, 1).drops())); + observe("reserve_inc_xrp", static_cast(fees.increment.drops())); + } + + // Seconds since the last validated ledger closed. + auto const age = app.getLedgerMaster().getValidatedLedgerAge(); + observe("ledger_age_seconds", static_cast(age.count())); + + // Transaction rate from the open ledger's tx count. + // OpenView::txCount() tracks transactions in the current + // open ledger; dividing by the ledger age gives an + // approximate rate. + auto const& openLedger = app.getOpenLedger(); + auto const txInLedger = openLedger.current()->txCount(); + auto const ageVal = age.count(); + if (ageVal > 0) + { + observe( + "transaction_rate", + static_cast(txInLedger) / static_cast(ageVal)); + } + else + { + observe("transaction_rate", 0.0); + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerStateTrackingGauge() +{ + // --- Task 7.12: State tracking gauges --- + stateTrackingGauge_ = + meter_->CreateDoubleObservableGauge("state_tracking", "Node state and mode tracking"); + stateTrackingGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // State value: 0-4 from OperatingMode, 5=validating, 6=proposing. + auto const mode = app.getOPs().getOperatingMode(); + auto stateValue = static_cast(mode); + + // If FULL, refine using consensus info for validating/proposing. + if (mode == OperatingMode::FULL) + { + auto const info = app.getOPs().getConsensusInfo(); + if (info.isMember("proposing") && info["proposing"].asBool()) + { + stateValue = 6.0; + } + else if (info.isMember("validating") && info["validating"].asBool()) + { + stateValue = 5.0; + } + } + observe("state_value", stateValue); + + // Time spent in the current operating mode, sourced from + // NetworkOPs' StateAccounting via a lightweight accessor. + auto const stateDurUs = app.getOPs().getServerStateDurationUs(); + observe( + "time_in_current_state_seconds", static_cast(stateDurUs.count()) / 1e6); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerStorageDetailGauge() +{ + // --- Task 7.13: Storage detail gauges --- + // Reports NuDB on-disk size via the NodeStore JSON counters interface. + storageDetailGauge_ = + meter_->CreateInt64ObservableGauge("storage_detail", "Storage detail metrics"); + storageDetailGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // NuDB on-disk size reported by the NodeStore backend. + // getStoreSize() returns the total bytes stored. + observe("nudb_bytes", static_cast(app.getNodeStore().getStoreSize())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); +} + +void +MetricsRegistry::registerValidationAgreementGauge() +{ + // --- Task 7.15: Validation agreement gauges --- + // Reports rolling-window agreement percentages and counts from + // ValidationTracker. reconcile() is called at the start of the + // callback so that pending ledger events are resolved before the + // window data is read (the callback fires every ~10 s from the + // PeriodicExportingMetricReader thread). + validationAgreementGauge_ = meter_->CreateDoubleObservableGauge( + "validation_agreement", "Validation agreement percentages and counts (1h/24h windows)"); + validationAgreementGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + + try + { + // Reconcile pending events before reading window data. + self->validationTracker_.reconcile(); + + auto observe = [&](char const* name, double value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + observe("agreement_pct_1h", self->validationTracker_.agreementPct1h()); + observe("agreement_pct_24h", self->validationTracker_.agreementPct24h()); + observe( + "agreements_1h", static_cast(self->validationTracker_.agreements1h())); + observe("missed_1h", static_cast(self->validationTracker_.missed1h())); + observe( + "agreements_24h", + static_cast(self->validationTracker_.agreements24h())); + observe("missed_24h", static_cast(self->validationTracker_.missed24h())); + + // 7-day window (matches external xrpl-validator-dashboard). + observe("agreement_pct_7d", self->validationTracker_.agreementPct7d()); + observe( + "agreements_7d", static_cast(self->validationTracker_.agreements7d())); + observe("missed_7d", static_cast(self->validationTracker_.missed7d())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); +} + +void +MetricsRegistry::registerValidationTotalsCounters() +{ + // Lifetime validation agreement/miss counters. + // + // These are monotonic ObservableCounters (not the sync Counters they used + // to be): a Prometheus _total must never decrease, but ValidationTracker's + // NET totals are non-monotonic (a late repair decrements the net miss + // count). We therefore observe the tracker's GROSS lifetime tallies, which + // count each ledger once at first classification and are never adjusted on + // repair (initial-classification semantics — see ValidationTracker). The + // repaired/agreement view remains available from validation_agreement. + // + // reconcile() is called first so pending events are resolved before the + // tallies are read; the callback fires every ~10 s from the + // PeriodicExportingMetricReader thread. + validationAgreementsObservable_ = meter_->CreateInt64ObservableCounter( + "validation_agreements_total", + "Lifetime validations that initially agreed with network consensus"); + validationAgreementsObservable_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + try + { + self->validationTracker_.reconcile(); + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(self->validationTracker_.totalAgreementsEver())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); + + validationMissedObservable_ = meter_->CreateInt64ObservableCounter( + "validation_missed_total", "Lifetime validations that initially missed network consensus"); + validationMissedObservable_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + try + { + self->validationTracker_.reconcile(); + opentelemetry::nostd::get>>(result) + ->Observe(static_cast(self->validationTracker_.totalMissedEver())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip on error. + } + }, + this); +} + +#endif // XRPL_ENABLE_TELEMETRY + +// ----------------------------------------------------------------- +// External dashboard parity counter increments (Task 7.14) +// ----------------------------------------------------------------- + +void +MetricsRegistry::incrementLedgersClosed() +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && ledgersClosedCounter_) + ledgersClosedCounter_->Add(1); +#endif +} + +void +MetricsRegistry::incrementValidationsSent() +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && validationsSentCounter_) + validationsSentCounter_->Add(1); +#endif +} + +void +MetricsRegistry::incrementValidationsChecked() +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && validationsCheckedCounter_) + validationsCheckedCounter_->Add(1); +#endif +} + +void +MetricsRegistry::incrementStateChanges() +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && stateChangesCounter_) + stateChangesCounter_->Add(1); +#endif +} + +void +MetricsRegistry::incrementLedgerHistoryMismatch(std::string_view reason) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && ledgerHistoryMismatchCounter_) + ledgerHistoryMismatchCounter_->Add(1, {{"reason", std::string(reason)}}); +#endif +} + +void +MetricsRegistry::incrementTxqExpired() +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && txqExpiredCounter_) + txqExpiredCounter_->Add(1); +#endif +} + +void +MetricsRegistry::incrementTxqDropped(std::string_view reason) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (enabled_ && txqDroppedCounter_) + txqDroppedCounter_->Add(1, {{"reason", std::string(reason)}}); +#endif +} + +} // namespace xrpl::telemetry diff --git a/src/xrpld/telemetry/MetricsRegistry.h b/src/xrpld/telemetry/MetricsRegistry.h new file mode 100644 index 00000000000..40bf38abbf9 --- /dev/null +++ b/src/xrpld/telemetry/MetricsRegistry.h @@ -0,0 +1,834 @@ +#pragma once + +/** + * Central OTel Metrics Registry for xrpld. + * + * Owns all OpenTelemetry metric instruments (counters, histograms, + * observable gauges) that are NOT already covered by the beast::insight + * StatsD pipeline. The instruments are created once at startup and polled + * by the OTel PeriodicExportingMetricReader at a configurable interval + * (default 10 s). + * + * When XRPL_ENABLE_TELEMETRY is **not** defined, this class compiles to a + * lightweight no-op: every public method is an empty inline. + * + * Dependency / ownership diagram (ASCII): + * + * Application + * | + * +-- MetricsRegistry (unique_ptr, created in setup(), started/stopped with telemetry) + * | + * +-- OTel MeterProvider (owns reader + exporter) + * | | + * | +-- PeriodicExportingMetricReader + * | +-- OtlpHttpMetricExporter + * | + * +-- Counters / Histograms (synchronous instruments) + * | +-- rpc_method_started_total + * | +-- rpc_method_finished_total + * | +-- rpc_method_errored_total + * | +-- rpc_method_us (Histogram) + * | +-- job_queued_total{job_type,handler} + * | +-- job_started_total{job_type,handler} + * | +-- job_finished_total{job_type,handler} + * | +-- job_queued_us{job_type,handler} (Histogram) + * | +-- job_running_us{job_type,handler} (Histogram) + * | +-- ledgers_closed_total + * | +-- validations_sent_total + * | +-- validations_checked_total + * | +-- state_changes_total + * | +-- ledger_history_mismatch_total{reason} + * | +-- txq_expired_total + * | +-- txq_dropped_total{reason} + * | + * +-- ValidationTracker (validation agreement tracker) + * | + * +-- Observable Gauges (async callbacks, polled by reader) + * +-- Cache hit rates (SLE, ledger, AL) + * +-- TreeNode / FullBelow sizes + * +-- TxQ metrics + * +-- CountedObject counts + * +-- Load factor breakdown + * +-- NodeStore I/O gauges + * +-- Server info (state, uptime, peers, consensus) + * +-- Build info (version label) + * +-- Complete ledger ranges (start/end pairs) + * +-- DB metrics (storage KB, fetch rate) + * +-- Validator health (amend blocked, UNL, quorum) + * +-- Peer quality (P90 latency, version spread) + * +-- Reduce-relay efficiency (selected/suppressed peers) + * +-- Ledger economy (fees, reserves, age) + * +-- State tracking (mode value, time in state) + * +-- Storage detail (NuDB sizes) + * +-- Validation agreement (1h/24h pct, counts) + * +-- jq_trans_overflow_total (observed from Overlay) + * + * Control-flow for async gauges: + * + * PeriodicExportingMetricReader (background thread, 10 s tick) + * | + * v + * OTel SDK invokes registered ObservableGauge callbacks + * | + * v + * Each callback reads current value from Application services + * (e.g. app.getTxQ().getMetrics(), app.getFeeTrack().getLoadFactor()) + * | + * v + * Result set is exported via OTLP/HTTP to the collector + * + * Control-flow for synchronous instruments: + * + * PerfLogImp::rpcStart/rpcEnd/jobQueue/jobStart/jobFinish + * | + * v + * MetricsRegistry::recordRpc*(method, ...) / recordJob*(type, ...) + * | + * v + * OTel Counter::Add() or Histogram::Record() + * | + * v + * Periodically flushed by the MetricReader + * + * Example usage: + * + * @code + * // In Application::setup(), after telemetry_ is created: + * metricsRegistry_ = std::make_unique( + * telemetry_->isEnabled(), app, journal); + * metricsRegistry_->start(setup.exporterEndpoint); + * + * // In PerfLogImp::rpcStart(): + * if (auto* mr = app_.getMetricsRegistry()) + * mr->recordRpcStarted("server_info"); + * + * // In PerfLogImp::rpcEnd(): + * if (auto* mr = app_.getMetricsRegistry()) + * { + * mr->recordRpcFinished("server_info", durationUs); + * // or: mr->recordRpcErrored("server_info", durationUs); + * } + * + * // In PerfLogImp::jobQueue(). The second argument is the addJob name; + * // it is sanitised internally into the bounded `handler` label. + * if (auto* mr = app_.getMetricsRegistry()) + * mr->recordJobQueued("ledgerData", "ProcessLData"); + * + * // Shutdown: + * metricsRegistry_->stop(); + * @endcode + * + * Caveats: + * - The MetricsRegistry must be created AFTER the Telemetry object because + * it reads isEnabled() to decide whether to initialize the OTel SDK. + * - Observable gauge callbacks capture a reference to the Application; the + * Application must outlive the MetricsRegistry (guaranteed because + * MetricsRegistry is stopped before Application teardown). + * - If a new CountedObject type is added, it will NOT appear automatically + * in the object_count gauge; the callback iterates a fixed list. + * - Adding a new synchronous instrument requires updating both the header + * and the .cpp, then calling the new record*() method from the + * instrumentation site. + */ + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifdef XRPL_ENABLE_TELEMETRY +#include +#include +#include +#include +#include +#endif + +namespace xrpl { + +class ServiceRegistry; + +namespace telemetry { + +/** + * Central OpenTelemetry metric registry. + * + * Owns all OTel instruments (counters, histograms, observable gauges) + * that are not covered by the beast::insight StatsD pipeline. See the + * file-level header comment above for the full dependency diagram, + * gauge domain list, and usage examples. + * + * Class / collaborator diagram (ASCII): + * + * +-----------------+ +-------------------+ + * | Application |------->| MetricsRegistry | + * +-----------------+ +-------------------+ + * | | | + * creates/owns v v v + * +-----------+ +---------+ +-------------------+ + * | Meter | | Counter | | ValidationTracker | + * | Provider | | /Hist. | | (rolling windows) | + * +-----------+ +---------+ +-------------------+ + * | + * v + * Periodic reader thread (~10 s) + * -> ObservableGauge callbacks + * -> OTLP/HTTP export + * + * @note Thread safety: + * - The recordRpc, recordJob, and increment methods are invoked + * from xrpld hot paths. OTel Counter::Add() and + * Histogram::Record() are documented thread-safe, and + * null-guard checks protect uninitialized instruments. + * - ObservableGauge callbacks run on the OTel SDK background + * reader thread (~10 s tick), concurrently with writers. + * Each callback reads only lock-protected or atomic state + * from Application services and wraps the body in a + * catch-all try block so a transient failure never crashes + * the reader thread. + * - ValidationTracker protects its rolling windows internally. + * - start() and stop() are NOT thread-safe with each other and + * must be called from the single Application lifecycle + * thread. + * + * @note Lifetime: + * - Must be constructed AFTER telemetry_ (reads isEnabled()). + * - Must be stopped BEFORE Application services it observes are + * destroyed; the Application owns it via unique_ptr so normal + * teardown guarantees this. + * + * @note Extending: + * - Adding a new CountedObject type is auto-picked up by the + * object_count gauge via iteration. + * - Adding a new SYNCHRONOUS instrument (counter/histogram): prefer the + * XRPL_METRIC_* call-site macros in MetricMacros.h -- no header/cpp + * edit needed. Fall back to a dedicated member + init line + record + * method (the pattern below) only when the metric needs to be read + * back by other code (e.g. ValidationTracker-style accumulation) or + * needs a custom histogram bucket View (see MetricMacros.h Limitation + * 2 in tasks/metric-macro-plan.md). + * - Adding a new OBSERVABLE gauge still requires eager central + * registration -- pull-model instruments cannot be lazily created. + */ +class MetricsRegistry +{ +public: + /** + * Construct a MetricsRegistry. + * + * @param enabled Whether OTel metric export is active. When false, + * all methods become no-ops. + * @param app Reference to the ServiceRegistry (Application) for + * reading current metric values in gauge callbacks. + * @param journal Journal for log output. + */ + MetricsRegistry(bool enabled, ServiceRegistry& app, beast::Journal journal); + + ~MetricsRegistry(); + + /** + * Non-copyable, non-movable. + */ + MetricsRegistry(MetricsRegistry const&) = delete; + MetricsRegistry& + operator=(MetricsRegistry const&) = delete; + + /** + * Initialize the OTel metrics pipeline and register all instruments. + * + * @param endpoint OTLP/HTTP endpoint URL for metric export + * (e.g. "http://localhost:4318/v1/metrics"). + * @param instanceId Value for the service.instance.id resource + * attribute. When non-empty, Prometheus metrics + * carry a service_instance_id label for per-node + * filtering. + */ + void + start(std::string const& endpoint, std::string const& instanceId = {}); + + /** + * Detach all ObservableGauge callbacks so they no-op on the next + * reader-thread tick. + * + * Must be called BEFORE any Application service that the callbacks + * read (nodeStore, overlay, networkOPs, ledgerMaster, etc.) is + * stopped. The flag is checked with acquire ordering at the top of + * every callback; together with the release store here it + * guarantees that once `detachCallbacks()` returns, no subsequent + * callback invocation will dereference an already-stopped service. + * + * Idempotent. Safe to call multiple times. Safe to call before + * `start()` (has no effect). The actual SDK-level provider + * shutdown still happens in `stop()`. + */ + void + detachCallbacks() noexcept; + + /** + * Flush pending metrics and shut down the pipeline. + * + * @pre `detachCallbacks()` should have been called earlier in the + * shutdown sequence; otherwise there is a narrow race between + * the final reader-thread tick and the destruction of + * Application services that the gauge callbacks read from. + */ + void + stop(); + + /** + * @return true if the registry is actively exporting metrics. + */ + [[nodiscard]] bool + isEnabled() const noexcept + { + return enabled_; + } + + // ----------------------------------------------------------------- + // Synchronous instrument recording (called from PerfLog hot paths) + // ----------------------------------------------------------------- + + /** + * Record an RPC method call start. + * @param method The RPC method name (e.g. "server_info"). + */ + void + recordRpcStarted(std::string_view method); + + /** + * Record an RPC method call completion. + * @param method The RPC method name. + * @param durationUs Execution time in microseconds. + */ + void + recordRpcFinished(std::string_view method, std::int64_t durationUs); + + /** + * Record an RPC method call error. + * @param method The RPC method name. + * @param durationUs Execution time in microseconds. + */ + void + recordRpcErrored(std::string_view method, std::int64_t durationUs); + + /** + * The `handler` label value used for any job name that fails the + * sanitiser's all-ASCII-letters rule. + * + * Public because both sanitiseHandler() and its unit tests must agree + * on the exact fallback token; a test asserting against its own copy + * of the string would not catch a change made here. + * + * Declared as std::string_view rather than the `constexpr char k[]` + * form used for instrument names in MetricsRegistry.cpp: this value is + * *returned* by sanitiseHandler(), whose return type is + * std::string_view, and is compared against std::string_view in tests. + * Matching the type avoids array-to-pointer decay and a needless + * strlen at each use. + */ + static constexpr std::string_view kHandlerOther{"other"}; + + /** + * Reduce a job name to a bounded-cardinality `handler` label value. + * + * A job type can have several producers — both `RcvGetLedger` and + * `RcvGetObjByHash` run as `JtLedgerReq` — so `job_type` alone cannot + * attribute a latency spike to one of them. The job name can, but it + * cannot be used raw: two names embed a ledger sequence number + * (`"Pub" + std::to_string(seq)` in LedgerPersistence.cpp and + * `"OB" + std::to_string(...)` in OrderBookDBImpl.cpp), which would + * mint a fresh Prometheus series for every ledger. + * + * The rule is therefore: keep the name only when it is non-empty and + * every character is an ASCII letter; otherwise return `"other"`. + * Both dynamic names always contain digits, so they always fold to + * `"other"`, while every all-letter name is a compile-time literal. + * The label domain is thus a function of the literals present in the + * source — 43 names plus `"other"` at the time of writing — and + * cannot grow at runtime. A name added later that does not satisfy + * the rule degrades to `"other"` rather than becoming unbounded, + * which is a stronger guarantee than an allowlist that would have to + * be maintained by hand. + * + * Defined inline so unit tests can call it without linking the rest + * of the registry: in a telemetry-enabled build MetricsRegistry.cpp + * is not compiled into the test binary, so an out-of-line definition + * would be unreachable from tests. Being inline also makes it usable + * regardless of XRPL_ENABLE_TELEMETRY. + * + * @param name The job name as passed to JobQueue::addJob. + * @return @p name when it is non-empty and all ASCII letters, else + * kHandlerOther. + * + * @note Pure and reentrant: holds no state, performs no I/O, and is + * safe to call concurrently from any thread. + * @note The letter test is an explicit ASCII range check rather than + * std::isalpha, which classifies by the current C locale. A + * locale-dependent test could admit non-ASCII bytes and so + * weaken the cardinality bound this function exists to provide. + * @note When the name is kept, the returned view aliases @p name, so + * it must not outlive the caller's buffer. The kHandlerOther case + * returns a view of a static constant and is always valid. + * + * Example: + * @code + * sanitiseHandler("RcvGetObjByHash"); // "RcvGetObjByHash" + * sanitiseHandler("Pub94512331"); // kHandlerOther (digits) + * sanitiseHandler(""); // kHandlerOther (empty) + * @endcode + */ + [[nodiscard]] static constexpr std::string_view + sanitiseHandler(std::string_view name) noexcept + { + auto const isAsciiLetter = [](char const c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + }; + + if (name.empty() || !std::ranges::all_of(name, isAsciiLetter)) + return kHandlerOther; + + return name; + } + + /** + * Record a job enqueued event. + * @param jobType The job type name (e.g. "ledgerData"). + * @param jobName The addJob name, reduced to a bounded `handler` + * label by sanitiseHandler(). Distinguishes producers + * that share a job type. + */ + void + recordJobQueued(std::string_view jobType, std::string_view jobName); + + /** + * Record a job start event. + * @param jobType The job type name. + * @param jobName The addJob name; see recordJobQueued(). + * @param queuedDurUs Time the job spent waiting in the queue (us). + */ + void + recordJobStarted(std::string_view jobType, std::string_view jobName, std::int64_t queuedDurUs); + + /** + * Record a job finish event. + * @param jobType The job type name. + * @param jobName The addJob name; see recordJobQueued(). + * @param runningDurUs Execution time in microseconds. + */ + void + recordJobFinished( + std::string_view jobType, + std::string_view jobName, + std::int64_t runningDurUs); + + // ----------------------------------------------------------------- + // External dashboard parity counters (Tasks 7.9-7.14) + // ----------------------------------------------------------------- + + /** + * Increment the ledgers_closed_total counter. + * + * @note Currently has no callers: the ledgers_closed_total counter is + * incremented at its consensus call site via the XRPL_METRIC_COUNTER_INC + * macro (see MetricMacros.h). This method and its eagerly-created + * counter are retained as a fallback and are slated for removal in a + * separate cleanup once the macro path has proven out. + */ + void + incrementLedgersClosed(); + + /** + * Increment the validations_sent_total counter. + * Called from RCLConsensus::Adaptor::validate() when a validation + * is produced and broadcast. + */ + void + incrementValidationsSent(); + + /** + * Increment the validations_checked_total counter. + * Called from NetworkOPs::recvValidation() when a network validation + * is received and checked. + */ + void + incrementValidationsChecked(); + + /** + * Increment the state_changes_total counter. + * Called from NetworkOPsImp::setMode() when the server operating mode + * changes (e.g. CONNECTED -> SYNCING -> TRACKING -> FULL). + */ + void + incrementStateChanges(); + + /** + * Increment the ledger_history_mismatch_total counter for a reason. + * Called from LedgerHistory::handleMismatch() once the mismatch has + * been classified. The reason label turns fork diagnosis from a + * log-grep into a queryable time series. + * @param reason Classified mismatch cause (e.g. "prior_ledger", + * "close_time", "consensus_txset", "same_txset_diff_result", + * "unknown"). + */ + void + incrementLedgerHistoryMismatch(std::string_view reason); + + /** + * Increment the txq_expired_total counter. + * Called from TxQ::processClosedLedger() for each queued transaction + * removed because its LastLedgerSequence has passed — submitters who + * under-bid the escalating fee and were never included. + */ + void + incrementTxqExpired(); + + /** + * Increment the txq_dropped_total{reason} counter. + * Called from TxQ::apply() when a transaction is refused admission to + * the queue (e.g. the queue is full). Distinct from expiry (already + * queued) and from jq_trans_overflow (job queue, not TxQ). + * @param reason Admission-control rejection cause (e.g. "queue_full"). + */ + void + incrementTxqDropped(std::string_view reason); + + /** + * Access the validation agreement tracker. + * Used by consensus and ledger hooks to record our validations and + * network validations so the tracker can compute agreement percentages. + * @return Reference to the internal ValidationTracker instance. + */ + ValidationTracker& + getValidationTracker() + { + return validationTracker_; + } + +#ifdef XRPL_ENABLE_TELEMETRY + /** + * Access the shared OTel Meter for call-site instrument creation. + * Used by the XRPL_METRIC_* macros (MetricMacros.h) so new synchronous + * counters/histograms can be declared at their call site instead of as + * MetricsRegistry members. Returns an empty (falsy) shared_ptr before + * start() has run or when disabled. + * @return The shared Meter, or empty if not yet started. + */ + opentelemetry::nostd::shared_ptr + meter() const noexcept + { + return meter_; + } +#endif + +private: + /** + * Master enable flag; when false all methods are no-ops. + */ + bool const enabled_; + + /** + * Tracks validation agreement between this node and the network. + * Lives outside the XRPL_ENABLE_TELEMETRY guard because it is + * always safe to record events; the gauge callback simply won't + * fire when telemetry is disabled. + */ + ValidationTracker validationTracker_; + +#ifdef XRPL_ENABLE_TELEMETRY + /** + * Reference to Application services for gauge callbacks. + * Only needed when OTel is compiled in, since observable gauge + * callbacks live entirely inside the XRPL_ENABLE_TELEMETRY guard. + */ + ServiceRegistry& app_; + + /** + * Journal for logging. + */ + beast::Journal const journal_; + + /** + * Set by detachCallbacks() during shutdown so every ObservableGauge + * callback returns early before reading Application services that + * may already be stopped. Checked with memory_order_acquire at the + * top of each callback to pair with the memory_order_release store + * in detachCallbacks(). + */ + std::atomic callbacksDetached_{false}; + + /** + * The SDK MeterProvider that owns the export pipeline. + */ + std::shared_ptr provider_; + + /** + * The Meter used to create all instruments. + */ + opentelemetry::nostd::shared_ptr meter_; + + // --- Synchronous instruments (RPC) --- + /** + * Counter: rpc_method_started_total{method=""} + */ + opentelemetry::nostd::unique_ptr> rpcStartedCounter_; + /** + * Counter: rpc_method_finished_total{method=""} + */ + opentelemetry::nostd::unique_ptr> rpcFinishedCounter_; + /** + * Counter: rpc_method_errored_total{method=""} + */ + opentelemetry::nostd::unique_ptr> rpcErroredCounter_; + /** + * Histogram: rpc_method_duration_us{method=""} + */ + opentelemetry::nostd::unique_ptr> + rpcDurationHistogram_; + + // --- Synchronous instruments (Job Queue) --- + // All five carry handler="" in addition to + // job_type, so producers that share a job type stay distinguishable. + /** + * Counter: job_queued_total{job_type="",handler=""} + */ + opentelemetry::nostd::unique_ptr> jobQueuedCounter_; + /** + * Counter: job_started_total{job_type="",handler=""} + */ + opentelemetry::nostd::unique_ptr> jobStartedCounter_; + /** + * Counter: job_finished_total{job_type="",handler=""} + */ + opentelemetry::nostd::unique_ptr> jobFinishedCounter_; + /** + * Histogram: job_queued_duration_us{job_type="",handler=""} + */ + opentelemetry::nostd::unique_ptr> + jobQueuedDurationHistogram_; + /** + * Histogram: job_running_duration_us{job_type="",handler=""} + */ + opentelemetry::nostd::unique_ptr> + jobRunningDurationHistogram_; + + // --- Observable gauges (registered via callbacks) --- + // Handles are stored so we can remove callbacks on shutdown. + /** + * Observable gauges for cache hit rates and sizes. + */ + opentelemetry::nostd::shared_ptr + cacheHitRateGauge_; + /** + * Observable gauges for TxQ metrics. + */ + opentelemetry::nostd::shared_ptr txqGauge_; + /** + * Observable gauges for counted object instances. + */ + opentelemetry::nostd::shared_ptr + objectCountGauge_; + /** + * Observable gauges for load factor breakdown. + */ + opentelemetry::nostd::shared_ptr loadFactorGauge_; + /** + * Observable gauges for NodeStore write_load and read_queue. + */ + opentelemetry::nostd::shared_ptr nodeStoreGauge_; + /** + * Observable gauge for server-level health metrics (state, uptime, peers, etc.). + */ + opentelemetry::nostd::shared_ptr serverInfoGauge_; + /** + * Observable gauge for build version info (label-based, value=1). + */ + opentelemetry::nostd::shared_ptr buildInfoGauge_; + /** + * Observable gauge for complete ledger range start/end pairs. + */ + opentelemetry::nostd::shared_ptr + completeLedgersGauge_; + /** + * Observable gauge for database sizes and historical fetch rate. + */ + opentelemetry::nostd::shared_ptr dbMetricsGauge_; + + // --- External dashboard parity gauges (Tasks 7.9-7.13) --- + /** + * Observable gauge for validator health indicators (amendment blocked, + * UNL blocked, quorum, UNL expiry). + */ + opentelemetry::nostd::shared_ptr + validatorHealthGauge_; + /** + * Observable gauge for peer network quality metrics (P90 latency, + * insane peer count, version spread, upgrade recommendation). + */ + opentelemetry::nostd::shared_ptr + peerQualityGauge_; + /** + * Observable gauge for transaction reduce-relay efficiency (selected vs + * suppressed peers, feature-disabled peers, missing-tx frequency). + */ + opentelemetry::nostd::shared_ptr + reduceRelayGauge_; + /** + * Observable gauge for ledger economy metrics (base fee, reserve, + * reserve increment, ledger age). + */ + opentelemetry::nostd::shared_ptr + ledgerEconomyGauge_; + /** + * Observable gauge for node state tracking (operating mode value, + * time in current state). + */ + opentelemetry::nostd::shared_ptr + stateTrackingGauge_; + /** + * Observable gauge for storage detail metrics (NuDB on-disk size). + */ + opentelemetry::nostd::shared_ptr + storageDetailGauge_; + /** + * Observable gauge for validation agreement metrics (1h/24h percentages + * and counts from ValidationTracker). + */ + opentelemetry::nostd::shared_ptr + validationAgreementGauge_; + + // --- External dashboard parity counters (Task 7.14) --- + /** + * Counter: ledgers_closed_total — incremented each consensus round. + */ + opentelemetry::nostd::unique_ptr> + ledgersClosedCounter_; + /** + * Counter: validations_sent_total — incremented when this node sends a validation. + */ + opentelemetry::nostd::unique_ptr> + validationsSentCounter_; + /** + * Counter: validations_checked_total — incremented for each network validation + * received. + */ + opentelemetry::nostd::unique_ptr> + validationsCheckedCounter_; + /** + * Counter: state_changes_total — incremented on operating mode transitions. + */ + opentelemetry::nostd::unique_ptr> + stateChangesCounter_; + /** + * ObservableCounter: jq_trans_overflow_total — observed from + * Overlay::getJqTransOverflow() (cumulative overflow tally owned by the overlay). + */ + opentelemetry::nostd::shared_ptr + jqTransOverflowObservable_; + /** + * Counter: ledger_history_mismatch_total{reason} — incremented per classified + * built-vs-validated ledger mismatch. + */ + opentelemetry::nostd::unique_ptr> + ledgerHistoryMismatchCounter_; + /** + * Counter: txq_expired_total — incremented per transaction expired out of the + * transaction queue. + */ + opentelemetry::nostd::unique_ptr> txqExpiredCounter_; + /** + * Counter: txq_dropped_total{reason} — incremented when a transaction is refused + * admission to the queue. + */ + opentelemetry::nostd::unique_ptr> txqDroppedCounter_; + /** + * ObservableCounter: validation_agreements_total — observed from + * ValidationTracker::totalAgreementsEver() (monotonic gross lifetime + * tally, initial-classification semantics). + */ + opentelemetry::nostd::shared_ptr + validationAgreementsObservable_; + /** + * ObservableCounter: validation_missed_total — observed from + * ValidationTracker::totalMissedEver() (monotonic gross lifetime tally, + * initial-classification semantics). + */ + opentelemetry::nostd::shared_ptr + validationMissedObservable_; + + /** + * Build the OTLP/HTTP exporter, periodic reader, resource attributes and + * histogram views, then create the MeterProvider and meter. Extracted + * from start() to keep each function under the 80-line limit. + * + * @param endpoint OTLP/HTTP metrics endpoint URL. + * @param instanceId service.instance.id resource attribute (may be empty). + */ + void + initExporterAndProvider(std::string const& endpoint, std::string const& instanceId); + + /** + * Create the synchronous instruments (RPC and job-queue counters and + * histograms, plus the external dashboard parity counters). Extracted + * from start() to keep each function under the 80-line limit. + */ + void + initSyncInstruments(); + + /** + * Register all observable gauge callbacks with the OTel SDK. + * Dispatches to one helper per metric domain so that each helper + * stays well under the 80-line-per-function limit. + */ + void + registerAsyncGauges(); + + // Per-domain gauge registration helpers. Each creates its instrument + // and attaches a single ObservableGauge callback that reads current + // values from Application services. The callbacks run on the OTel + // PeriodicExportingMetricReader background thread (~10 s tick). + void + registerCacheHitRateGauge(); // Task 9.2 + void + registerTxqGauge(); // Task 9.3 + void + registerObjectCountGauge(); // Task 9.6 + void + registerLoadFactorGauge(); // Task 9.7 + void + registerNodeStoreGauge(); // Task 9.1 + void + registerServerInfoGauge(); // Task 9.7a + void + registerBuildInfoGauge(); // Task 9.7b + void + registerCompleteLedgersGauge(); // Task 9.7c + void + registerDbMetricsGauge(); // Task 9.7d + void + registerValidatorHealthGauge(); // Task 7.9 + void + registerPeerQualityGauge(); // Task 7.10 + void + registerReduceRelayGauge(); // Reduce-relay efficiency + void + registerLedgerEconomyGauge(); // Task 7.11 + void + registerStateTrackingGauge(); // Task 7.12 + void + registerStorageDetailGauge(); // Task 7.13 + void + registerValidationAgreementGauge(); // Task 7.15 + void + registerValidationTotalsCounters(); // gap-fill: lifetime agree/miss _total +#endif // XRPL_ENABLE_TELEMETRY +}; + +} // namespace telemetry +} // namespace xrpl diff --git a/src/xrpld/telemetry/PropagationHelpers.h b/src/xrpld/telemetry/PropagationHelpers.h new file mode 100644 index 00000000000..ce64733d82f --- /dev/null +++ b/src/xrpld/telemetry/PropagationHelpers.h @@ -0,0 +1,63 @@ +#pragma once + +/** + * Helpers for injecting trace context into protobuf messages. + * + * Bridges the gap between SpanGuard (which hides OTel types) and the + * protobuf TraceContext message used for cross-node propagation. + * + * Dependency diagram: + * + * SpanGuard::getTraceBytes() protocol::TraceContext (proto) + * \ / + * +--- TraceBytes -----+ + * | | + * injectSpanContext(span, proto) + * + * @note When XRPL_ENABLE_TELEMETRY is disabled, getTraceBytes() returns + * {.valid=false}, so injectSpanContext becomes a no-op with zero overhead. + * + * Usage: + * @code + * // Send side — inject from a SpanGuard reference: + * protocol::TMTransaction tx; + * // ... populate tx fields ... + * injectSpanContext(mySpanGuard, *tx.mutable_trace_context()); + * overlay.relay(txID, tx, toSkip); + * @endcode + * + * @see ConsensusReceiveTracing.h and TxTracing.h for receive-side + * extraction helpers. + * @see TraceContextPropagator.h for low-level OTel context serialization. + */ + +#include +#include + +namespace xrpl::telemetry { + +/** + * Inject trace context from an active SpanGuard into a protobuf + * TraceContext message for cross-node propagation. + * + * Reads the span's trace_id, span_id, and trace_flags via + * getTraceBytes() and writes them into the protobuf fields. + * Safe to call from any thread that holds a reference to the span. + * No-op if the span is null or inactive. + * + * @param span The active SpanGuard whose context to propagate. + * @param proto The protobuf TraceContext to populate. + */ +inline void +injectSpanContext(SpanGuard const& span, protocol::TraceContext& proto) +{ + auto const bytes = span.getTraceBytes(); + if (!bytes.valid) + return; + + proto.set_trace_id(bytes.traceId.data(), bytes.traceId.size()); + proto.set_span_id(bytes.spanId.data(), bytes.spanId.size()); + proto.set_trace_flags(bytes.traceFlags); +} + +} // namespace xrpl::telemetry diff --git a/src/xrpld/telemetry/TxSpanNames.h b/src/xrpld/telemetry/TxSpanNames.h new file mode 100644 index 00000000000..2a6aa9824c7 --- /dev/null +++ b/src/xrpld/telemetry/TxSpanNames.h @@ -0,0 +1,129 @@ +#pragma once + +/** + * Compile-time span name constants for transaction tracing. + * + * Used by PeerImp (overlay) and NetworkOPs (app) for transaction + * lifecycle spans. Built on StaticStr/join() from SpanNames.h. + * + * Span hierarchy (cross-node propagation): + * + * Node A (sender) Node B (receiver) + * +---------------------+ +---------------------+ + * | tx.process | protobuf | tx.receive | + * | injectSpanContext | ---------> | txReceiveSpan() | + * | (PropagationHelp.) | trace_ctx | extracts parent | + * +---------------------+ +---------------------+ + */ + +#include + +namespace xrpl::telemetry::tx_span { + +// ===== Span prefixes ======================================================= + +namespace prefix { +/** + * "tx" — root prefix for transaction lifecycle spans. + */ +inline constexpr auto tx = seg::tx; +} // namespace prefix + +// ===== Span operation suffixes ============================================= + +namespace op { +inline constexpr auto receive = makeStr("receive"); +inline constexpr auto process = makeStr("process"); +} // namespace op + +// ===== Full span names (prefix.op) ========================================= + +inline constexpr auto receive = join(prefix::tx, op::receive); +inline constexpr auto process = join(prefix::tx, op::process); + +// ===== Attribute keys ====================================================== + +namespace attr { +/** + * Canonical shared constants (defined in SpanNames.h). + */ +using ::xrpl::telemetry::attr::peerId; +using ::xrpl::telemetry::attr::txHash; + +/** + * "current_ledger_seq" — the open/current ledger index at the time the tx is + * received or submitted (the ledger being worked on), NOT an established ledger. + * No parent-ledger hash here: these entry points have no ledger view. + */ +using ::xrpl::telemetry::attr::currentLedgerSeq; + +/** + * "local" — whether tx originated locally. + */ +inline constexpr auto local = makeStr("local"); +/** + * "path" — sync or async processing path. + */ +inline constexpr auto path = makeStr("path"); +/** + * "suppressed" — whether tx was suppressed as duplicate. + */ +inline constexpr auto suppressed = makeStr("suppressed"); +/** + * "tx_status" — domain-qualified (collides with rpc_status, txq_status). + */ +inline constexpr auto txStatus = makeStr("tx_status"); +/** + * "peer_version" — version of peer that sent the tx. + */ +inline constexpr auto peerVersion = makeStr("peer_version"); +/** + * "tx_type" — transaction type name (e.g., "Payment", "OfferCreate"). + */ +inline constexpr auto txType = makeStr("tx_type"); +/** + * "fee" — transaction fee in drops. + */ +inline constexpr auto fee = makeStr("fee"); +/** + * "sequence" — transaction sequence number. + */ +inline constexpr auto sequence = makeStr("sequence"); +/** + * "ter_result" — engine result code after application. + */ +inline constexpr auto terResult = makeStr("ter_result"); +/** + * "applied" — whether the transaction was applied to the ledger. + */ +inline constexpr auto applied = makeStr("applied"); +} // namespace attr + +// ===== Attribute values ==================================================== + +namespace val { +inline constexpr auto sync = makeStr("sync"); +inline constexpr auto async = makeStr("async"); +inline constexpr auto knownBad = makeStr("known_bad"); +/** + * Transaction was suppressed via HashRouter (duplicate, not flagged bad). + */ +inline constexpr auto suppressed = makeStr("suppressed"); +/** + * Transaction was rejected because it carried tfInnerBatchTxn, which + * must never appear in network-relayed traffic. + */ +inline constexpr auto rejectedInnerBatch = makeStr("rejected_inner_batch"); +/** + * Transaction was dropped because the validated ledger is too old to + * confidently apply new transactions (server is out of sync). + */ +inline constexpr auto droppedNoSync = makeStr("dropped_no_sync"); +/** + * Transaction was dropped because the local job queue for jtTRANSACTION + * is at MAX_TRANSACTIONS — backpressure on the receive side. + */ +inline constexpr auto droppedQueueFull = makeStr("dropped_queue_full"); +} // namespace val + +} // namespace xrpl::telemetry::tx_span diff --git a/src/xrpld/telemetry/TxTracing.h b/src/xrpld/telemetry/TxTracing.h new file mode 100644 index 00000000000..02900fa9b42 --- /dev/null +++ b/src/xrpld/telemetry/TxTracing.h @@ -0,0 +1,69 @@ +#pragma once + +/** + * Helper functions for creating transaction trace spans. + * + * Encapsulates the logic for creating SpanGuard instances with + * hash-derived trace IDs and optional protobuf parent extraction. + * Call sites in PeerImp and NetworkOPs stay simple one-liners. + * + * When XRPL_ENABLE_TELEMETRY is not defined, the functions return + * no-op SpanGuard instances (zero overhead, zero dependencies). + */ + +#include + +#include +#include +#include +#include + +#include + +namespace xrpl::telemetry { + +/** + * Create a "tx.receive" span for a transaction received from a peer. + * trace_id is derived from txID[0:16]. If the incoming message carries + * a protobuf TraceContext with a valid span_id, it is used as the + * parent to preserve relay ordering. + */ +inline SpanGuard +txReceiveSpan(uint256 const& txID, [[maybe_unused]] protocol::TMTransaction const& msg) +{ +#ifdef XRPL_ENABLE_TELEMETRY + if (msg.has_trace_context()) + { + auto const& tc = msg.trace_context(); + // Only the span_id is taken from the peer here; the trace_id is + // derived locally from txID, so validate the span_id alone. + if (tc.has_span_id() && isValidSpanId(tc.span_id())) + { + return SpanGuard::hashSpan( + TraceCategory::Transactions, + tx_span::receive, + txID.data(), + txID.kBytes, + reinterpret_cast(tc.span_id().data()), + tc.span_id().size(), + tc.has_trace_flags() ? static_cast(tc.trace_flags()) + : std::uint8_t{0}); + } + } +#endif + return SpanGuard::hashSpan( + TraceCategory::Transactions, tx_span::receive, txID.data(), txID.kBytes); +} + +/** + * Create a "tx.process" span for transaction processing in NetworkOPs. + * trace_id is derived from txID[0:16]. + */ +inline SpanGuard +txProcessSpan(uint256 const& txID) +{ + return SpanGuard::hashSpan( + TraceCategory::Transactions, tx_span::process, txID.data(), txID.kBytes); +} + +} // namespace xrpl::telemetry diff --git a/src/xrpld/telemetry/ValidationTracker.h b/src/xrpld/telemetry/ValidationTracker.h new file mode 100644 index 00000000000..ac80f5cdf50 --- /dev/null +++ b/src/xrpld/telemetry/ValidationTracker.h @@ -0,0 +1,410 @@ +#pragma once + +/** + * @file ValidationTracker.h + * Standalone validation agreement tracker for telemetry. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace xrpl::telemetry { + +/** + * Tracks whether this validator's validations agree with network consensus, + * maintaining rolling 1-hour and 24-hour windows plus lifetime totals. + * + * The tracker operates by recording two independent events per ledger: + * 1. "We validated" -- our node published a validation for a ledger hash. + * 2. "Network validated" -- the network reached consensus on a ledger hash. + * + * After a configurable grace period (kGracePeriod), the reconcile() method + * compares the two flags. If both are set the ledger is counted as an + * "agreement"; otherwise it is a "miss". A late-repair mechanism allows a + * miss to be upgraded to an agreement if matching evidence arrives within + * kLateRepairWindow. + * + * Architecture / dependency diagram: + * @code + * +--------------------------+ + * | ConsensusAdapter / | + * | ValidatorSite | + * | (callers) | + * +---+-------------+-------+ + * | | + * | recordOur | recordNetwork + * | Validation | Validation + * v v + * +---------------------------+ + * | ValidationTracker | + * |---------------------------| + * | pending_ (hash_map) |----> LedgerEvent per hash + * | window1h_ (deque) |----> WindowEvent sliding window + * | window24h_ (deque) |----> WindowEvent sliding window + * | atomic totals | + * +---------------------------+ + * | + * | reconcile() called periodically + * v + * agreement / miss counters updated + * @endcode + * + * Usage -- basic recording and querying: + * @code + * xrpl::telemetry::ValidationTracker tracker; + * + * // On local validation: + * tracker.recordOurValidation(ledgerHash, seq); + * + * // On network consensus: + * tracker.recordNetworkValidation(ledgerHash, seq); + * + * // Periodically (e.g. every few seconds): + * tracker.reconcile(); + * + * // Query agreement percentage: + * double pct = tracker.agreementPct1h(); + * @endcode + * + * Usage -- edge case with late arrival: + * @code + * xrpl::telemetry::ValidationTracker tracker; + * + * // Network validates first, our validation arrives late: + * tracker.recordNetworkValidation(hash, seq); + * tracker.reconcile(); // initially counted as a miss + * + * // Late local validation arrives within repair window: + * tracker.recordOurValidation(hash, seq); + * tracker.reconcile(); // repaired to agreement + * @endcode + * + * @note Thread-safety: all public methods are thread-safe. The pending_ + * map and sliding-window deques are protected by mutex_. Lifetime totals + * use std::atomic for lock-free reads. + */ +class ValidationTracker +{ +public: + /** + * Monotonic clock used for all internal timestamps. + */ + using Clock = std::chrono::steady_clock; + + /** + * Time point type from the monotonic clock. + */ + using TimePoint = Clock::time_point; + + /** + * Record that this node sent a validation for the given ledger. + * @param ledgerHash Hash of the ledger we validated. + * @param seq Ledger sequence number. + */ + void + recordOurValidation(uint256 const& ledgerHash, LedgerIndex seq); + + /** + * Record that the network reached consensus on the given ledger. + * @param ledgerHash Hash of the network-validated ledger. + * @param seq Ledger sequence number. + */ + void + recordNetworkValidation(uint256 const& ledgerHash, LedgerIndex seq); + + /** + * Reconcile pending ledger events whose grace period has elapsed. + * Should be called periodically (e.g. every few seconds). Moves + * reconciled events into the sliding windows and updates totals. + * Also performs late-repair and eviction of stale data. + */ + void + reconcile(); + + /** + * @name Rolling-window percentage getters + */ + /** @{ */ + + /** + * Agreement percentage over the last 1 hour. + * @return Percentage [0.0, 100.0], or 0.0 if no data. + */ + double + agreementPct1h() const; + + /** + * Agreement percentage over the last 24 hours. + * @return Percentage [0.0, 100.0], or 0.0 if no data. + */ + double + agreementPct24h() const; + + /** + * Agreement percentage over the last 7 days. + * @return Percentage [0.0, 100.0], or 0.0 if no data. + */ + double + agreementPct7d() const; + + /** @} */ + + /** + * @name Rolling-window count getters + */ + /** @{ */ + + /** + * Number of agreements in the 1-hour window. + */ + uint64_t + agreements1h() const; + + /** + * Number of misses in the 1-hour window. + */ + uint64_t + missed1h() const; + + /** + * Number of agreements in the 24-hour window. + */ + uint64_t + agreements24h() const; + + /** + * Number of misses in the 24-hour window. + */ + uint64_t + missed24h() const; + + /** + * Number of agreements in the 7-day window. + */ + uint64_t + agreements7d() const; + + /** + * Number of misses in the 7-day window. + */ + uint64_t + missed7d() const; + + /** @} */ + + /** + * @name Lifetime totals (atomic, lock-free reads) + */ + /** @{ */ + + /** + * Total agreements since process start. + */ + uint64_t + totalAgreements() const; + + /** + * Total misses since process start. + */ + uint64_t + totalMissed() const; + + /** + * Lifetime agreements counted at first classification only. + * + * @note Unlike totalAgreements(), this is strictly monotonic: it is + * incremented only when a ledger is first reconciled as an agreement and + * is never adjusted by a late repair. It backs the monotonic Prometheus + * counter validation_agreements_total. See the counting-semantics + * note in detail/ValidationTracker.cpp. + */ + uint64_t + totalAgreementsEver() const; + + /** + * Lifetime misses counted at first classification only. + * + * @note Unlike totalMissed(), this is strictly monotonic: it is + * incremented only when a ledger is first reconciled as a miss and is + * never decremented by a late repair. It backs the monotonic Prometheus + * counter validation_missed_total. See the counting-semantics note + * in detail/ValidationTracker.cpp. + */ + uint64_t + totalMissedEver() const; + + /** + * Total validations this node sent. + */ + uint64_t + totalValidationsSent() const; + + /** + * Total network validations observed for comparison. + */ + uint64_t + totalValidationsChecked() const; + + /** @} */ + +private: + /** + * Per-ledger tracking state held in the pending map. + */ + struct LedgerEvent + { + uint256 ledgerHash; ///< Ledger hash being tracked. + LedgerIndex seq{0}; ///< Ledger sequence number. + TimePoint recordTime; ///< Time the event was first recorded. + bool weValidated = false; ///< True if we sent a validation. + bool networkValidated = false; ///< True if network reached consensus. + bool reconciled = false; ///< True once grace period elapsed. + bool agreed = false; ///< True if both flags set at reconcile. + }; + + /** + * Lightweight event stored in the sliding-window deques. + */ + struct WindowEvent + { + TimePoint time; ///< When the event was reconciled. + uint256 ledgerHash; ///< Ledger hash for late-repair matching. + bool agreed{false}; ///< Whether this was an agreement. + }; + + /** + * Grace period before reconciling a ledger event. + */ + static constexpr auto kGracePeriod = std::chrono::seconds(8); + + /** + * Window during which a missed event can be repaired. + */ + static constexpr auto kLateRepairWindow = std::chrono::minutes(5); + + /** + * Maximum number of pending (unreconciled + recently reconciled) events. + */ + static constexpr std::size_t kMaxPendingEvents = 1000; + + /** + * Duration of the short rolling window. + */ + static constexpr auto kWindow1h = std::chrono::hours(1); + + /** + * Duration of the long rolling window. + */ + static constexpr auto kWindow24h = std::chrono::hours(24); + + /** + * Duration of the extended rolling window (7 days). + */ + static constexpr auto kWindow7d = std::chrono::hours(168); + + /** + * Protects pending_, window1h_, window24h_, and window7d_. + */ + mutable std::mutex mutex_; + + /** + * Pending ledger events indexed by ledger hash. + */ + hash_map pending_; + + /** + * Sliding window of reconciled events (last 1 hour). + */ + std::deque window1h_; + + /** + * Sliding window of reconciled events (last 24 hours). + */ + std::deque window24h_; + + /** + * Sliding window of reconciled events (last 7 days). + */ + std::deque window7d_; + + /** + * Lifetime count of agreements (net: incremented on agree, also on + * repair). May be read via totalAgreements(); feeds the windowed gauge. + */ + std::atomic totalAgreements_{0}; + + /** + * Lifetime count of misses (net: incremented on miss, decremented on + * repair). NON-monotonic. May be read via totalMissed(). + */ + std::atomic totalMissed_{0}; + + /** + * Monotonic "gross" lifetime tallies for the Prometheus _total counters. + * + * Counting decision (initial-classification only): each reconciled + * ledger is counted exactly once, at its first classification, into + * exactly one of the two tallies below. A later late-repair + * (miss -> agreement) does NOT move either tally. This keeps both + * strictly monotonic (a Prometheus _total must never decrease) and + * additive: totalAgreementsGross_ + totalMissedGross_ == ledgers + * reconciled. The repaired/agreement view is still available from the + * windowed gauge (validation_agreement) and the net totals above. + */ + + /** + * Monotonic lifetime initial agreements; backs + * validation_agreements_total. Never adjusted on repair. + */ + std::atomic totalAgreementsGross_{0}; + + /** + * Monotonic lifetime initial misses; backs validation_missed_total. + * Never decremented on repair. + */ + std::atomic totalMissedGross_{0}; + + /** + * Lifetime count of validations this node sent. + */ + std::atomic totalValidationsSent_{0}; + + /** + * Lifetime count of network validations observed. + */ + std::atomic totalValidationsChecked_{0}; + + /** + * Remove entries older than their respective window durations. + * @param now Current time point. + */ + void + evictStaleWindows(TimePoint now); + + /** + * Remove reconciled pending entries older than the late-repair window. + * Also trims the map if it exceeds kMaxPendingEvents. + * @param now Current time point. + */ + void + evictOldPending(TimePoint now); + + /** + * Scan a window deque and flip the first non-agreed entry matching + * the given ledger hash to agreed. + * @param window The sliding-window deque to repair. + * @param hash Ledger hash to match. + */ + static void + repairWindowEntry(std::deque& window, uint256 const& hash); +}; + +} // namespace xrpl::telemetry diff --git a/src/xrpld/telemetry/detail/ValidationTracker.cpp b/src/xrpld/telemetry/detail/ValidationTracker.cpp new file mode 100644 index 00000000000..c7f9c599bce --- /dev/null +++ b/src/xrpld/telemetry/detail/ValidationTracker.cpp @@ -0,0 +1,297 @@ +/** + * @file ValidationTracker.cpp + * Implementation of the ValidationTracker class. + */ + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace xrpl::telemetry { + +void +ValidationTracker::recordOurValidation(uint256 const& ledgerHash, LedgerIndex seq) +{ + std::scoped_lock const lock(mutex_); + auto& evt = pending_[ledgerHash]; + if (evt.recordTime == TimePoint{}) + { + // First time seeing this ledger hash -- initialize. + evt.ledgerHash = ledgerHash; + evt.seq = seq; + evt.recordTime = Clock::now(); + } + evt.weValidated = true; + totalValidationsSent_.fetch_add(1, std::memory_order_relaxed); +} + +void +ValidationTracker::recordNetworkValidation(uint256 const& ledgerHash, LedgerIndex seq) +{ + std::scoped_lock const lock(mutex_); + auto& evt = pending_[ledgerHash]; + if (evt.recordTime == TimePoint{}) + { + evt.ledgerHash = ledgerHash; + evt.seq = seq; + evt.recordTime = Clock::now(); + } + evt.networkValidated = true; + totalValidationsChecked_.fetch_add(1, std::memory_order_relaxed); +} + +void +ValidationTracker::reconcile() +{ + std::scoped_lock const lock(mutex_); + auto const now = Clock::now(); + + for (auto& [hash, evt] : pending_) + { + if (!evt.reconciled && (now - evt.recordTime) >= kGracePeriod) + { + // Initial reconciliation after grace period. + evt.reconciled = true; + evt.agreed = evt.weValidated && evt.networkValidated; + + if (evt.agreed) + { + totalAgreements_.fetch_add(1, std::memory_order_relaxed); + // Gross tally: count the initial agreement once. See the + // counting-decision note below (repair branch). + totalAgreementsGross_.fetch_add(1, std::memory_order_relaxed); + } + else + { + totalMissed_.fetch_add(1, std::memory_order_relaxed); + // Gross tally: count the initial miss once. See the + // counting-decision note below (repair branch). + totalMissedGross_.fetch_add(1, std::memory_order_relaxed); + } + + WindowEvent const we{.time = now, .ledgerHash = evt.ledgerHash, .agreed = evt.agreed}; + window1h_.push_back(we); + window24h_.push_back(we); + window7d_.push_back(we); + } + else if ( + evt.reconciled && !evt.agreed && evt.weValidated && evt.networkValidated && + (now - evt.recordTime) <= kLateRepairWindow) + { + // Late repair: was a miss, now both flags set. Adjust the NET + // totals (used by the windowed agreement gauge) so the live view + // reflects the repair. + evt.agreed = true; + totalMissed_.fetch_sub(1, std::memory_order_relaxed); + totalAgreements_.fetch_add(1, std::memory_order_relaxed); + + // Counting decision (initial-classification only): the gross + // tallies (totalAgreementsGross_ / totalMissedGross_) that back the + // monotonic Prometheus _total counters are deliberately NOT touched + // here. Each ledger is counted once, at first classification; a + // repair must not decrement missed (a _total may never decrease) + // nor add a second agreement (which would double-count the ledger). + + // Flip the corresponding window entries from miss to agreement. + repairWindowEntry(window1h_, evt.ledgerHash); + repairWindowEntry(window24h_, evt.ledgerHash); + repairWindowEntry(window7d_, evt.ledgerHash); + } + } + + evictStaleWindows(now); + evictOldPending(now); +} + +void +ValidationTracker::evictStaleWindows(TimePoint now) +{ + auto const cutoff1h = now - kWindow1h; + while (!window1h_.empty() && window1h_.front().time < cutoff1h) + window1h_.pop_front(); + + auto const cutoff24h = now - kWindow24h; + while (!window24h_.empty() && window24h_.front().time < cutoff24h) + window24h_.pop_front(); + + auto const cutoff7d = now - kWindow7d; + while (!window7d_.empty() && window7d_.front().time < cutoff7d) + window7d_.pop_front(); +} + +void +ValidationTracker::evictOldPending(TimePoint now) +{ + auto const cutoff = now - kLateRepairWindow; + for (auto it = pending_.begin(); it != pending_.end();) + { + if (it->second.reconciled && it->second.recordTime < cutoff) + { + it = pending_.erase(it); + } + else + { + ++it; + } + } + + // Hard trim if still over limit. The loop above already removed every + // reconciled entry older than the late-repair window, so here we drop + // any remaining reconciled entry as a last resort. + if (pending_.size() > kMaxPendingEvents) + { + for (auto it = pending_.begin(); + it != pending_.end() && pending_.size() > kMaxPendingEvents;) + { + if (it->second.reconciled) + { + it = pending_.erase(it); + } + else + { + ++it; + } + } + } +} + +double +ValidationTracker::agreementPct1h() const +{ + std::scoped_lock const lock(mutex_); + if (window1h_.empty()) + return 0.0; + auto const agreed = static_cast( + std::count_if(window1h_.begin(), window1h_.end(), [](auto const& e) { return e.agreed; })); + return (agreed / static_cast(window1h_.size())) * 100.0; +} + +double +ValidationTracker::agreementPct24h() const +{ + std::scoped_lock const lock(mutex_); + if (window24h_.empty()) + return 0.0; + auto const agreed = static_cast(std::count_if( + window24h_.begin(), window24h_.end(), [](auto const& e) { return e.agreed; })); + return (agreed / static_cast(window24h_.size())) * 100.0; +} + +uint64_t +ValidationTracker::agreements1h() const +{ + std::scoped_lock const lock(mutex_); + return static_cast( + std::count_if(window1h_.begin(), window1h_.end(), [](auto const& e) { return e.agreed; })); +} + +uint64_t +ValidationTracker::missed1h() const +{ + std::scoped_lock const lock(mutex_); + return static_cast( + std::count_if(window1h_.begin(), window1h_.end(), [](auto const& e) { return !e.agreed; })); +} + +uint64_t +ValidationTracker::agreements24h() const +{ + std::scoped_lock const lock(mutex_); + return static_cast(std::count_if( + window24h_.begin(), window24h_.end(), [](auto const& e) { return e.agreed; })); +} + +uint64_t +ValidationTracker::missed24h() const +{ + std::scoped_lock const lock(mutex_); + return static_cast(std::count_if( + window24h_.begin(), window24h_.end(), [](auto const& e) { return !e.agreed; })); +} + +double +ValidationTracker::agreementPct7d() const +{ + std::scoped_lock const lock(mutex_); + if (window7d_.empty()) + return 0.0; + auto const agreed = static_cast( + std::count_if(window7d_.begin(), window7d_.end(), [](auto const& e) { return e.agreed; })); + return (agreed / static_cast(window7d_.size())) * 100.0; +} + +uint64_t +ValidationTracker::agreements7d() const +{ + std::scoped_lock const lock(mutex_); + return static_cast( + std::count_if(window7d_.begin(), window7d_.end(), [](auto const& e) { return e.agreed; })); +} + +uint64_t +ValidationTracker::missed7d() const +{ + std::scoped_lock const lock(mutex_); + return static_cast( + std::count_if(window7d_.begin(), window7d_.end(), [](auto const& e) { return !e.agreed; })); +} + +uint64_t +ValidationTracker::totalAgreements() const +{ + return totalAgreements_.load(std::memory_order_relaxed); +} + +uint64_t +ValidationTracker::totalMissed() const +{ + return totalMissed_.load(std::memory_order_relaxed); +} + +uint64_t +ValidationTracker::totalAgreementsEver() const +{ + return totalAgreementsGross_.load(std::memory_order_relaxed); +} + +uint64_t +ValidationTracker::totalMissedEver() const +{ + return totalMissedGross_.load(std::memory_order_relaxed); +} + +uint64_t +ValidationTracker::totalValidationsSent() const +{ + return totalValidationsSent_.load(std::memory_order_relaxed); +} + +uint64_t +ValidationTracker::totalValidationsChecked() const +{ + return totalValidationsChecked_.load(std::memory_order_relaxed); +} + +void +ValidationTracker::repairWindowEntry(std::deque& window, uint256 const& hash) +{ + // Scan backwards since late repairs target recently added entries. + for (auto& event : std::views::reverse(window)) + { + if (!event.agreed && event.ledgerHash == hash) + { + event.agreed = true; + return; + } + } +} + +} // namespace xrpl::telemetry diff --git a/tasks/fix-validation-checks.md b/tasks/fix-validation-checks.md new file mode 100644 index 00000000000..ea33fd92235 --- /dev/null +++ b/tasks/fix-validation-checks.md @@ -0,0 +1,169 @@ +# Fix Telemetry Validation Checks + +## Context + +The CI pipeline infrastructure is fully operational (build + deploy + run). However, +the `validate_telemetry.py` validation suite fails 35 checks due to mismatches between +what the validation expects and what the telemetry stack actually produces. These fall +into 4 categories. + +CI run: https://github.com/XRPLF/rippled/actions/runs/23026466191 + +--- + +## Category 1: StatsD Metrics — 0 Series (25 failures) + +**Symptoms:** + +``` +[FAIL] metric.statsd_gauges.xrpld_LedgerMaster_Validated_Ledger_Age: 0 series +[FAIL] metric.statsd_counters.xrpld_rpc_requests: 0 series +[FAIL] metric.statsd_histograms.xrpld_rpc_time: 0 series +[FAIL] metric.overlay_traffic.xrpld_total_Bytes_In: 0 series +[FAIL] metric.phase9_nodestore.xrpld_nodestore_reads_total: 0 series +... (25 total) +``` + +**Root Cause:** Two issues compounding: + +1. **StatsD receiver is commented out** in `otel-collector-config.yaml` (lines 39-54). + The collector config was updated to expect native OTLP metrics from beast::insight + (comment: "StatsD UDP port removed — beast::insight now uses native OTLP"), but + the validation harness configures xrpld nodes with `server=statsd`. + +2. **Metric name mismatch:** The `expected_metrics.json` expects StatsD-style metric + names (e.g., `xrpld_LedgerMaster_Validated_Ledger_Age`). When using `server=otel`, + beast::insight emits OTLP metrics which may have different names/structure. + +**Fix Options (pick one):** + +- **Option A (recommended):** Change the node config in `run-full-validation.sh` from + `server=statsd` to `server=otel` (line 255), remove the `address=127.0.0.1:8125` line, + then update `expected_metrics.json` with the actual OTLP metric names. This aligns with + the collector config's OTLP-first design and avoids re-enabling the StatsD receiver. + +- **Option B:** Uncomment the StatsD receiver in `otel-collector-config.yaml`, add + `statsd` to the metrics pipeline receivers list, and keep node config as `server=statsd`. + Simpler but goes against the migration to native OTLP. + +**Investigation needed for Option A:** + +- Run xrpld locally with `server=otel`, query Prometheus, and capture the actual OTLP + metric names to update `expected_metrics.json`. + +**Files to modify:** + +- `docker/telemetry/workload/run-full-validation.sh` — change `[insight]` section +- `docker/telemetry/workload/expected_metrics.json` — update metric names for OTLP +- `docker/telemetry/workload/validate_telemetry.py` — may need metric query adjustments + +--- + +## Category 2: Missing Spans — tx.process, tx.receive (2 failures) + +**Symptoms:** + +``` +[FAIL] span.tx.process: tx.process: 0 traces (expected > 0) +[FAIL] span.tx.receive: tx.receive: 0 traces (expected > 0) +``` + +**Root Cause:** The span names exist in the code: + +- `src/xrpld/app/misc/NetworkOPs.cpp:1228` — `XRPL_TRACE_TX("tx.process")` +- `src/xrpld/overlay/detail/PeerImp.cpp:1273` — `XRPL_TRACE_TX("tx.receive")` + +Likely causes (investigate in order): + +1. **Batch delay:** The 2-second batch delay (`batch_delay_ms=2000`) plus 30s propagation + wait may not be enough if these spans are created late in the workload. +2. **Code path not triggered:** `tx.process` fires in `NetworkOPs::processTransaction()`. + The tx_submitter submits via RPC `submit` command which calls this path. But if the + transactions fail validation before reaching `processTransaction()`, no span is emitted. +3. **Span naming mismatch:** The validation queries Tempo for exact operation name + `tx.process`. Verify Tempo stores the span with this exact name. + +**Investigation:** + +- Check the tx_submitter output in CI logs — are transactions actually succeeding? +- Query Tempo API locally for all span names to see what's actually emitted. + +**Files to modify:** + +- Possibly `docker/telemetry/workload/validate_telemetry.py` — adjust timing/queries +- Possibly `docker/telemetry/workload/run-full-validation.sh` — increase propagation wait + +--- + +## Category 3: Span Hierarchy — rpc.request -> rpc.process (1 failure) + +**Symptoms:** + +``` +[FAIL] span.hierarchy.rpc.request->rpc.process: rpc.process not found in rpc.request traces +``` + +**Root Cause:** The validator fetches traces containing `rpc.request` from Tempo and +checks if any child span is named `rpc.process`. Both spans are emitted (they pass +individual checks), but the parent-child relationship isn't established. + +**Investigation:** + +- Check `src/xrpld/rpc/detail/ServerHandler.cpp` — `rpc.request` (line 271) and + `rpc.process` (line 573) are in the same file. Verify that `rpc.process` is created + as a child of `rpc.request` (i.e., its parent context is set). +- The issue may be that `rpc.process` creates a new root span instead of linking to the + `rpc.request` span context. + +**Files to modify:** + +- Possibly `src/xrpld/rpc/detail/ServerHandler.cpp` — fix span parenting +- OR `docker/telemetry/workload/validate_telemetry.py` — if hierarchy check logic is wrong + +--- + +## Category 4: Dashboard 404s (5 failures) + +**Symptoms:** + +``` +[FAIL] dashboard.xrpld-statsd-node-health: HTTP 404 +[FAIL] dashboard.xrpld-statsd-network: HTTP 404 +[FAIL] dashboard.xrpld-statsd-rpc: HTTP 404 +[FAIL] dashboard.xrpld-statsd-overlay-detail: HTTP 404 +[FAIL] dashboard.xrpld-statsd-ledger-sync: HTTP 404 +``` + +**Root Cause:** Dashboard UIDs were renamed from `xrpld-statsd-*` to `xrpld-system-*` +but `expected_metrics.json` still references the old names. + +**Actual UIDs in `docker/telemetry/grafana/dashboards/`:** + +| Expected (in expected_metrics.json) | Actual (in dashboard JSON) | +| ----------------------------------- | ----------------------------- | +| `xrpld-statsd-node-health` | `xrpld-system-node-health` | +| `xrpld-statsd-network` | `xrpld-system-network` | +| `xrpld-statsd-rpc` | `xrpld-system-rpc` | +| `xrpld-statsd-overlay-detail` | `xrpld-system-overlay-detail` | +| `xrpld-statsd-ledger-sync` | `xrpld-system-ledger-sync` | + +**Fix:** Update the 5 UIDs in `expected_metrics.json` → `grafana_dashboards.uids[]`. + +**Files to modify:** + +- `docker/telemetry/workload/expected_metrics.json` — update dashboard UIDs + +--- + +## Execution Order + +1. **Category 4 (Dashboard UIDs)** — trivial rename, no investigation needed +2. **Category 1 (StatsD/OTLP metrics)** — requires investigation to choose Option A vs B + and capture actual metric names +3. **Category 2 (Missing tx spans)** — requires investigation into transaction code paths +4. **Category 3 (Span hierarchy)** — requires investigation into span context propagation + +## Branch + +All changes go on: `pratik/otel-phase10-workload-validation` +Worktree: `/tmp/otel-phase10-iter`